extern "C" {
typedef unsigned int size_t;
typedef __builtin_va_list __gnuc_va_list;
typedef unsigned char __u_char;
typedef unsigned short int __u_short;
typedef unsigned int __u_int;
typedef unsigned long int __u_long;
typedef signed char __int8_t;
typedef unsigned char __uint8_t;
typedef signed short int __int16_t;
typedef unsigned short int __uint16_t;
typedef signed int __int32_t;
typedef unsigned int __uint32_t;
__extension__ typedef signed long long int __int64_t;
__extension__ typedef unsigned long long int __uint64_t;
typedef __int8_t __int_least8_t;
typedef __uint8_t __uint_least8_t;
typedef __int16_t __int_least16_t;
typedef __uint16_t __uint_least16_t;
typedef __int32_t __int_least32_t;
typedef __uint32_t __uint_least32_t;
typedef __int64_t __int_least64_t;
typedef __uint64_t __uint_least64_t;
__extension__ typedef long long int __quad_t;
__extension__ typedef unsigned long long int __u_quad_t;
__extension__ typedef long long int __intmax_t;
__extension__ typedef unsigned long long int __uintmax_t;
__extension__ typedef __uint64_t __dev_t;
__extension__ typedef unsigned int __uid_t;
__extension__ typedef unsigned int __gid_t;
__extension__ typedef unsigned long int __ino_t;
__extension__ typedef __uint64_t __ino64_t;
__extension__ typedef unsigned int __mode_t;
__extension__ typedef unsigned int __nlink_t;
__extension__ typedef long int __off_t;
__extension__ typedef __int64_t __off64_t;
__extension__ typedef int __pid_t;
__extension__ typedef struct { int __val[2]; } __fsid_t;
__extension__ typedef long int __clock_t;
__extension__ typedef unsigned long int __rlim_t;
__extension__ typedef __uint64_t __rlim64_t;
__extension__ typedef unsigned int __id_t;
__extension__ typedef long int __time_t;
__extension__ typedef unsigned int __useconds_t;
__extension__ typedef long int __suseconds_t;
__extension__ typedef __int64_t __suseconds64_t;
__extension__ typedef int __daddr_t;
__extension__ typedef int __key_t;
__extension__ typedef int __clockid_t;
__extension__ typedef void * __timer_t;
__extension__ typedef long int __blksize_t;
__extension__ typedef long int __blkcnt_t;
__extension__ typedef __int64_t __blkcnt64_t;
__extension__ typedef unsigned long int __fsblkcnt_t;
__extension__ typedef __uint64_t __fsblkcnt64_t;
__extension__ typedef unsigned long int __fsfilcnt_t;
__extension__ typedef __uint64_t __fsfilcnt64_t;
__extension__ typedef int __fsword_t;
__extension__ typedef int __ssize_t;
__extension__ typedef long int __syscall_slong_t;
__extension__ typedef unsigned long int __syscall_ulong_t;
typedef __off64_t __loff_t;
typedef char *__caddr_t;
__extension__ typedef int __intptr_t;
__extension__ typedef unsigned int __socklen_t;
typedef int __sig_atomic_t;
__extension__ typedef __int64_t __time64_t;
typedef struct
{
int __count;
union
{
unsigned int __wch;
char __wchb[4];
} __value;
} __mbstate_t;
typedef struct _G_fpos_t
{
__off_t __pos;
__mbstate_t __state;
} __fpos_t;
typedef struct _G_fpos64_t
{
__off64_t __pos;
__mbstate_t __state;
} __fpos64_t;
struct _IO_FILE;
typedef struct _IO_FILE __FILE;
struct _IO_FILE;
typedef struct _IO_FILE FILE;
struct _IO_FILE;
struct _IO_marker;
struct _IO_codecvt;
struct _IO_wide_data;
typedef void _IO_lock_t;
struct _IO_FILE
{
int _flags;
char *_IO_read_ptr;
char *_IO_read_end;
char *_IO_read_base;
char *_IO_write_base;
char *_IO_write_ptr;
char *_IO_write_end;
char *_IO_buf_base;
char *_IO_buf_end;
char *_IO_save_base;
char *_IO_backup_base;
char *_IO_save_end;
struct _IO_marker *_markers;
struct _IO_FILE *_chain;
int _fileno;
int _flags2;
__off_t _old_offset;
unsigned short _cur_column;
signed char _vtable_offset;
char _shortbuf[1];
_IO_lock_t *_lock;
__off64_t _offset;
struct _IO_codecvt *_codecvt;
struct _IO_wide_data *_wide_data;
struct _IO_FILE *_freeres_list;
void *_freeres_buf;
size_t __pad5;
int _mode;
char _unused2[15 * sizeof (int) - 4 * sizeof (void *) - sizeof (size_t)];
};
typedef __ssize_t cookie_read_function_t (void *__cookie, char *__buf,
size_t __nbytes);
typedef __ssize_t cookie_write_function_t (void *__cookie, const char *__buf,
size_t __nbytes);
typedef int cookie_seek_function_t (void *__cookie, __off64_t *__pos, int __w);
typedef int cookie_close_function_t (void *__cookie);
typedef struct _IO_cookie_io_functions_t
{
cookie_read_function_t *read;
cookie_write_function_t *write;
cookie_seek_function_t *seek;
cookie_close_function_t *close;
} cookie_io_functions_t;
typedef __gnuc_va_list va_list;
typedef __off_t off_t;
typedef __off64_t off64_t;
typedef __ssize_t ssize_t;
typedef __fpos_t fpos_t;
typedef __fpos64_t fpos64_t;
extern FILE *stdin;
extern FILE *stdout;
extern FILE *stderr;
extern int remove (const char *__filename) noexcept (true);
extern int rename (const char *__old, const char *__new) noexcept (true);
extern int renameat (int __oldfd, const char *__old, int __newfd,
const char *__new) noexcept (true);
extern int renameat2 (int __oldfd, const char *__old, int __newfd,
const char *__new, unsigned int __flags) noexcept (true);
extern int fclose (FILE *__stream);
extern FILE *tmpfile (void)
__attribute__ ((__malloc__)) __attribute__ ((__malloc__ (fclose, 1))) ;
extern FILE *tmpfile64 (void)
__attribute__ ((__malloc__)) __attribute__ ((__malloc__ (fclose, 1))) ;
extern char *tmpnam (char[20]) noexcept (true) ;
extern char *tmpnam_r (char __s[20]) noexcept (true) ;
extern char *tempnam (const char *__dir, const char *__pfx)
noexcept (true) __attribute__ ((__malloc__)) __attribute__ ((__malloc__ (__builtin_free, 1)));
extern int fflush (FILE *__stream);
extern int fflush_unlocked (FILE *__stream);
extern int fcloseall (void);
extern FILE *fopen (const char *__restrict __filename,
const char *__restrict __modes)
__attribute__ ((__malloc__)) __attribute__ ((__malloc__ (fclose, 1))) ;
extern FILE *freopen (const char *__restrict __filename,
const char *__restrict __modes,
FILE *__restrict __stream) ;
extern FILE *fopen64 (const char *__restrict __filename,
const char *__restrict __modes)
__attribute__ ((__malloc__)) __attribute__ ((__malloc__ (fclose, 1))) ;
extern FILE *freopen64 (const char *__restrict __filename,
const char *__restrict __modes,
FILE *__restrict __stream) ;
extern FILE *fdopen (int __fd, const char *__modes) noexcept (true)
__attribute__ ((__malloc__)) __attribute__ ((__malloc__ (fclose, 1))) ;
extern FILE *fopencookie (void *__restrict __magic_cookie,
const char *__restrict __modes,
cookie_io_functions_t __io_funcs) noexcept (true)
__attribute__ ((__malloc__)) __attribute__ ((__malloc__ (fclose, 1))) ;
extern FILE *fmemopen (void *__s, size_t __len, const char *__modes)
noexcept (true) __attribute__ ((__malloc__)) __attribute__ ((__malloc__ (fclose, 1))) ;
extern FILE *open_memstream (char **__bufloc, size_t *__sizeloc) noexcept (true)
__attribute__ ((__malloc__)) __attribute__ ((__malloc__ (fclose, 1))) ;
extern void setbuf (FILE *__restrict __stream, char *__restrict __buf) noexcept (true);
extern int setvbuf (FILE *__restrict __stream, char *__restrict __buf,
int __modes, size_t __n) noexcept (true);
extern void setbuffer (FILE *__restrict __stream, char *__restrict __buf,
size_t __size) noexcept (true);
extern void setlinebuf (FILE *__stream) noexcept (true);
extern int fprintf (FILE *__restrict __stream,
const char *__restrict __format, ...);
extern int printf (const char *__restrict __format, ...);
extern int sprintf (char *__restrict __s,
const char *__restrict __format, ...) noexcept (true);
extern int vfprintf (FILE *__restrict __s, const char *__restrict __format,
__gnuc_va_list __arg);
extern int vprintf (const char *__restrict __format, __gnuc_va_list __arg);
extern int vsprintf (char *__restrict __s, const char *__restrict __format,
__gnuc_va_list __arg) noexcept (true);
extern int snprintf (char *__restrict __s, size_t __maxlen,
const char *__restrict __format, ...)
noexcept (true) __attribute__ ((__format__ (__printf__, 3, 4)));
extern int vsnprintf (char *__restrict __s, size_t __maxlen,
const char *__restrict __format, __gnuc_va_list __arg)
noexcept (true) __attribute__ ((__format__ (__printf__, 3, 0)));
extern int vasprintf (char **__restrict __ptr, const char *__restrict __f,
__gnuc_va_list __arg)
noexcept (true) __attribute__ ((__format__ (__printf__, 2, 0))) ;
extern int __asprintf (char **__restrict __ptr,
const char *__restrict __fmt, ...)
noexcept (true) __attribute__ ((__format__ (__printf__, 2, 3))) ;
extern int asprintf (char **__restrict __ptr,
const char *__restrict __fmt, ...)
noexcept (true) __attribute__ ((__format__ (__printf__, 2, 3))) ;
extern int vdprintf (int __fd, const char *__restrict __fmt,
__gnuc_va_list __arg)
__attribute__ ((__format__ (__printf__, 2, 0)));
extern int dprintf (int __fd, const char *__restrict __fmt, ...)
__attribute__ ((__format__ (__printf__, 2, 3)));
extern int fscanf (FILE *__restrict __stream,
const char *__restrict __format, ...) ;
extern int scanf (const char *__restrict __format, ...) ;
extern int sscanf (const char *__restrict __s,
const char *__restrict __format, ...) noexcept (true);
extern int fscanf (FILE *__restrict __stream, const char *__restrict __format, ...) __asm__ ("" "__isoc99_fscanf")
;
extern int scanf (const char *__restrict __format, ...) __asm__ ("" "__isoc99_scanf")
;
extern int sscanf (const char *__restrict __s, const char *__restrict __format, ...) noexcept (true) __asm__ ("" "__isoc99_sscanf")
;
extern int vfscanf (FILE *__restrict __s, const char *__restrict __format,
__gnuc_va_list __arg)
__attribute__ ((__format__ (__scanf__, 2, 0))) ;
extern int vscanf (const char *__restrict __format, __gnuc_va_list __arg)
__attribute__ ((__format__ (__scanf__, 1, 0))) ;
extern int vsscanf (const char *__restrict __s,
const char *__restrict __format, __gnuc_va_list __arg)
noexcept (true) __attribute__ ((__format__ (__scanf__, 2, 0)));
extern int vfscanf (FILE *__restrict __s, const char *__restrict __format, __gnuc_va_list __arg) __asm__ ("" "__isoc99_vfscanf")
__attribute__ ((__format__ (__scanf__, 2, 0))) ;
extern int vscanf (const char *__restrict __format, __gnuc_va_list __arg) __asm__ ("" "__isoc99_vscanf")
__attribute__ ((__format__ (__scanf__, 1, 0))) ;
extern int vsscanf (const char *__restrict __s, const char *__restrict __format, __gnuc_va_list __arg) noexcept (true) __asm__ ("" "__isoc99_vsscanf")
__attribute__ ((__format__ (__scanf__, 2, 0)));
extern int fgetc (FILE *__stream);
extern int getc (FILE *__stream);
extern int getchar (void);
extern int getc_unlocked (FILE *__stream);
extern int getchar_unlocked (void);
extern int fgetc_unlocked (FILE *__stream);
extern int fputc (int __c, FILE *__stream);
extern int putc (int __c, FILE *__stream);
extern int putchar (int __c);
extern int fputc_unlocked (int __c, FILE *__stream);
extern int putc_unlocked (int __c, FILE *__stream);
extern int putchar_unlocked (int __c);
extern int getw (FILE *__stream);
extern int putw (int __w, FILE *__stream);
extern char *fgets (char *__restrict __s, int __n, FILE *__restrict __stream)
__attribute__ ((__access__ (__write_only__, 1, 2)));
extern char *fgets_unlocked (char *__restrict __s, int __n,
FILE *__restrict __stream)
__attribute__ ((__access__ (__write_only__, 1, 2)));
extern __ssize_t __getdelim (char **__restrict __lineptr,
size_t *__restrict __n, int __delimiter,
FILE *__restrict __stream) ;
extern __ssize_t getdelim (char **__restrict __lineptr,
size_t *__restrict __n, int __delimiter,
FILE *__restrict __stream) ;
extern __ssize_t getline (char **__restrict __lineptr,
size_t *__restrict __n,
FILE *__restrict __stream) ;
extern int fputs (const char *__restrict __s, FILE *__restrict __stream);
extern int puts (const char *__s);
extern int ungetc (int __c, FILE *__stream);
extern size_t fread (void *__restrict __ptr, size_t __size,
size_t __n, FILE *__restrict __stream) ;
extern size_t fwrite (const void *__restrict __ptr, size_t __size,
size_t __n, FILE *__restrict __s);
extern int fputs_unlocked (const char *__restrict __s,
FILE *__restrict __stream);
extern size_t fread_unlocked (void *__restrict __ptr, size_t __size,
size_t __n, FILE *__restrict __stream) ;
extern size_t fwrite_unlocked (const void *__restrict __ptr, size_t __size,
size_t __n, FILE *__restrict __stream);
extern int fseek (FILE *__stream, long int __off, int __whence);
extern long int ftell (FILE *__stream) ;
extern void rewind (FILE *__stream);
extern int fseeko (FILE *__stream, __off_t __off, int __whence);
extern __off_t ftello (FILE *__stream) ;
extern int fgetpos (FILE *__restrict __stream, fpos_t *__restrict __pos);
extern int fsetpos (FILE *__stream, const fpos_t *__pos);
extern int fseeko64 (FILE *__stream, __off64_t __off, int __whence);
extern __off64_t ftello64 (FILE *__stream) ;
extern int fgetpos64 (FILE *__restrict __stream, fpos64_t *__restrict __pos);
extern int fsetpos64 (FILE *__stream, const fpos64_t *__pos);
extern void clearerr (FILE *__stream) noexcept (true);
extern int feof (FILE *__stream) noexcept (true) ;
extern int ferror (FILE *__stream) noexcept (true) ;
extern void clearerr_unlocked (FILE *__stream) noexcept (true);
extern int feof_unlocked (FILE *__stream) noexcept (true) ;
extern int ferror_unlocked (FILE *__stream) noexcept (true) ;
extern void perror (const char *__s);
extern int fileno (FILE *__stream) noexcept (true) ;
extern int fileno_unlocked (FILE *__stream) noexcept (true) ;
extern int pclose (FILE *__stream);
extern FILE *popen (const char *__command, const char *__modes)
__attribute__ ((__malloc__)) __attribute__ ((__malloc__ (pclose, 1))) ;
extern char *ctermid (char *__s) noexcept (true)
__attribute__ ((__access__ (__write_only__, 1)));
extern char *cuserid (char *__s)
__attribute__ ((__access__ (__write_only__, 1)));
struct obstack;
extern int obstack_printf (struct obstack *__restrict __obstack,
const char *__restrict __format, ...)
noexcept (true) __attribute__ ((__format__ (__printf__, 2, 3)));
extern int obstack_vprintf (struct obstack *__restrict __obstack,
const char *__restrict __format,
__gnuc_va_list __args)
noexcept (true) __attribute__ ((__format__ (__printf__, 2, 0)));
extern void flockfile (FILE *__stream) noexcept (true);
extern int ftrylockfile (FILE *__stream) noexcept (true) ;
extern void funlockfile (FILE *__stream) noexcept (true);
extern int __uflow (FILE *);
extern int __overflow (FILE *, int);
extern __inline __attribute__ ((__gnu_inline__)) int
vprintf (const char *__restrict __fmt, __gnuc_va_list __arg)
{
return vfprintf (stdout, __fmt, __arg);
}
extern __inline __attribute__ ((__gnu_inline__)) int
getchar (void)
{
return getc (stdin);
}
extern __inline __attribute__ ((__gnu_inline__)) int
fgetc_unlocked (FILE *__fp)
{
return (__builtin_expect (((__fp)->_IO_read_ptr >= (__fp)->_IO_read_end), 0) ? __uflow (__fp) : *(unsigned char *) (__fp)->_IO_read_ptr++);
}
extern __inline __attribute__ ((__gnu_inline__)) int
getc_unlocked (FILE *__fp)
{
return (__builtin_expect (((__fp)->_IO_read_ptr >= (__fp)->_IO_read_end), 0) ? __uflow (__fp) : *(unsigned char *) (__fp)->_IO_read_ptr++);
}
extern __inline __attribute__ ((__gnu_inline__)) int
getchar_unlocked (void)
{
return (__builtin_expect (((stdin)->_IO_read_ptr >= (stdin)->_IO_read_end), 0) ? __uflow (stdin) : *(unsigned char *) (stdin)->_IO_read_ptr++);
}
extern __inline __attribute__ ((__gnu_inline__)) int
putchar (int __c)
{
return putc (__c, stdout);
}
extern __inline __attribute__ ((__gnu_inline__)) int
fputc_unlocked (int __c, FILE *__stream)
{
return (__builtin_expect (((__stream)->_IO_write_ptr >= (__stream)->_IO_write_end), 0) ? __overflow (__stream, (unsigned char) (__c)) : (unsigned char) (*(__stream)->_IO_write_ptr++ = (__c)));
}
extern __inline __attribute__ ((__gnu_inline__)) int
putc_unlocked (int __c, FILE *__stream)
{
return (__builtin_expect (((__stream)->_IO_write_ptr >= (__stream)->_IO_write_end), 0) ? __overflow (__stream, (unsigned char) (__c)) : (unsigned char) (*(__stream)->_IO_write_ptr++ = (__c)));
}
extern __inline __attribute__ ((__gnu_inline__)) int
putchar_unlocked (int __c)
{
return (__builtin_expect (((stdout)->_IO_write_ptr >= (stdout)->_IO_write_end), 0) ? __overflow (stdout, (unsigned char) (__c)) : (unsigned char) (*(stdout)->_IO_write_ptr++ = (__c)));
}
extern __inline __attribute__ ((__gnu_inline__)) __ssize_t
getline (char **__lineptr, size_t *__n, FILE *__stream)
{
return __getdelim (__lineptr, __n, '\n', __stream);
}
extern __inline __attribute__ ((__gnu_inline__)) int
__attribute__ ((__leaf__)) feof_unlocked (FILE *__stream) noexcept (true)
{
return (((__stream)->_flags & 0x0010) != 0);
}
extern __inline __attribute__ ((__gnu_inline__)) int
__attribute__ ((__leaf__)) ferror_unlocked (FILE *__stream) noexcept (true)
{
return (((__stream)->_flags & 0x0020) != 0);
}
}
namespace std
{
typedef unsigned int size_t;
typedef int ptrdiff_t;
typedef decltype(nullptr) nullptr_t;
#pragma GCC visibility push(default)
extern "C++" __attribute__ ((__noreturn__, __always_inline__))
inline void __terminate() noexcept
{
void terminate() noexcept __attribute__ ((__noreturn__));
terminate();
}
#pragma GCC visibility pop
}
namespace std
{
inline namespace __cxx11 __attribute__((__abi_tag__ ("cxx11"))) { }
}
namespace __gnu_cxx
{
inline namespace __cxx11 __attribute__((__abi_tag__ ("cxx11"))) { }
}
namespace std
{
#pragma GCC visibility push(default)
constexpr inline bool
__is_constant_evaluated() noexcept
{
return __builtin_is_constant_evaluated();
}
#pragma GCC visibility pop
}
extern "C++" {
namespace std __attribute__ ((__visibility__ ("default")))
{
struct __true_type { };
struct __false_type { };
template<bool>
struct __truth_type
{ typedef __false_type __type; };
template<>
struct __truth_type<true>
{ typedef __true_type __type; };
template<class _Sp, class _Tp>
struct __traitor
{
enum { __value = bool(_Sp::__value) || bool(_Tp::__value) };
typedef typename __truth_type<__value>::__type __type;
};
template<typename, typename>
struct __are_same
{
enum { __value = 0 };
typedef __false_type __type;
};
template<typename _Tp>
struct __are_same<_Tp, _Tp>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<typename _Tp>
struct __is_void
{
enum { __value = 0 };
typedef __false_type __type;
};
template<>
struct __is_void<void>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<typename _Tp>
struct __is_integer
{
enum { __value = 0 };
typedef __false_type __type;
};
template<>
struct __is_integer<bool>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_integer<char>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_integer<signed char>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_integer<unsigned char>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_integer<wchar_t>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_integer<char16_t>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_integer<char32_t>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_integer<short>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_integer<unsigned short>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_integer<int>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_integer<unsigned int>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_integer<long>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_integer<unsigned long>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_integer<long long>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_integer<unsigned long long>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<typename _Tp>
struct __is_floating
{
enum { __value = 0 };
typedef __false_type __type;
};
template<>
struct __is_floating<float>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_floating<double>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_floating<long double>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<typename _Tp>
struct __is_pointer
{
enum { __value = 0 };
typedef __false_type __type;
};
template<typename _Tp>
struct __is_pointer<_Tp*>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<typename _Tp>
struct __is_arithmetic
: public __traitor<__is_integer<_Tp>, __is_floating<_Tp> >
{ };
template<typename _Tp>
struct __is_scalar
: public __traitor<__is_arithmetic<_Tp>, __is_pointer<_Tp> >
{ };
template<typename _Tp>
struct __is_char
{
enum { __value = 0 };
typedef __false_type __type;
};
template<>
struct __is_char<char>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_char<wchar_t>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<typename _Tp>
struct __is_byte
{
enum { __value = 0 };
typedef __false_type __type;
};
template<>
struct __is_byte<char>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_byte<signed char>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<>
struct __is_byte<unsigned char>
{
enum { __value = 1 };
typedef __true_type __type;
};
enum class byte : unsigned char;
template<>
struct __is_byte<byte>
{
enum { __value = 1 };
typedef __true_type __type;
};
template<typename> struct iterator_traits;
template<typename _Tp>
struct __is_nonvolatile_trivially_copyable
{
enum { __value = __is_trivially_copyable(_Tp) };
};
template<typename _Tp>
struct __is_nonvolatile_trivially_copyable<volatile _Tp>
{
enum { __value = 0 };
};
template<typename _OutputIter, typename _InputIter>
struct __memcpyable
{
enum { __value = 0 };
};
template<typename _Tp>
struct __memcpyable<_Tp*, _Tp*>
: __is_nonvolatile_trivially_copyable<_Tp>
{ };
template<typename _Tp>
struct __memcpyable<_Tp*, const _Tp*>
: __is_nonvolatile_trivially_copyable<_Tp>
{ };
template<typename _Iter1, typename _Iter2>
struct __memcmpable
{
enum { __value = 0 };
};
template<typename _Tp>
struct __memcmpable<_Tp*, _Tp*>
: __is_nonvolatile_trivially_copyable<_Tp>
{ };
template<typename _Tp>
struct __memcmpable<const _Tp*, _Tp*>
: __is_nonvolatile_trivially_copyable<_Tp>
{ };
template<typename _Tp>
struct __memcmpable<_Tp*, const _Tp*>
: __is_nonvolatile_trivially_copyable<_Tp>
{ };
template<typename _Tp, bool _TreatAsBytes =
__is_byte<_Tp>::__value
>
struct __is_memcmp_ordered
{
static const bool __value = _Tp(-1) > _Tp(1);
};
template<typename _Tp>
struct __is_memcmp_ordered<_Tp, false>
{
static const bool __value = false;
};
template<typename _Tp, typename _Up, bool = sizeof(_Tp) == sizeof(_Up)>
struct __is_memcmp_ordered_with
{
static const bool __value = __is_memcmp_ordered<_Tp>::__value
&& __is_memcmp_ordered<_Up>::__value;
};
template<typename _Tp, typename _Up>
struct __is_memcmp_ordered_with<_Tp, _Up, false>
{
static const bool __value = false;
};
template<>
struct __is_memcmp_ordered_with<std::byte, std::byte, true>
{ static constexpr bool __value = true; };
template<typename _Tp, bool _SameSize>
struct __is_memcmp_ordered_with<_Tp, std::byte, _SameSize>
{ static constexpr bool __value = false; };
template<typename _Up, bool _SameSize>
struct __is_memcmp_ordered_with<std::byte, _Up, _SameSize>
{ static constexpr bool __value = false; };
template<typename _Tp>
struct __is_move_iterator
{
enum { __value = 0 };
typedef __false_type __type;
};
template<typename _Iterator>
inline _Iterator
__miter_base(_Iterator __it)
{ return __it; }
}
}
extern "C++" {
namespace __gnu_cxx __attribute__ ((__visibility__ ("default")))
{
template<bool, typename>
struct __enable_if
{ };
template<typename _Tp>
struct __enable_if<true, _Tp>
{ typedef _Tp __type; };
template<bool _Cond, typename _Iftrue, typename _Iffalse>
struct __conditional_type
{ typedef _Iftrue __type; };
template<typename _Iftrue, typename _Iffalse>
struct __conditional_type<false, _Iftrue, _Iffalse>
{ typedef _Iffalse __type; };
template<typename _Tp>
struct __add_unsigned
{
private:
typedef __enable_if<std::__is_integer<_Tp>::__value, _Tp> __if_type;
public:
typedef typename __if_type::__type __type;
};
template<>
struct __add_unsigned<char>
{ typedef unsigned char __type; };
template<>
struct __add_unsigned<signed char>
{ typedef unsigned char __type; };
template<>
struct __add_unsigned<short>
{ typedef unsigned short __type; };
template<>
struct __add_unsigned<int>
{ typedef unsigned int __type; };
template<>
struct __add_unsigned<long>
{ typedef unsigned long __type; };
template<>
struct __add_unsigned<long long>
{ typedef unsigned long long __type; };
template<>
struct __add_unsigned<bool>;
template<>
struct __add_unsigned<wchar_t>;
template<typename _Tp>
struct __remove_unsigned
{
private:
typedef __enable_if<std::__is_integer<_Tp>::__value, _Tp> __if_type;
public:
typedef typename __if_type::__type __type;
};
template<>
struct __remove_unsigned<char>
{ typedef signed char __type; };
template<>
struct __remove_unsigned<unsigned char>
{ typedef signed char __type; };
template<>
struct __remove_unsigned<unsigned short>
{ typedef short __type; };
template<>
struct __remove_unsigned<unsigned int>
{ typedef int __type; };
template<>
struct __remove_unsigned<unsigned long>
{ typedef long __type; };
template<>
struct __remove_unsigned<unsigned long long>
{ typedef long long __type; };
template<>
struct __remove_unsigned<bool>;
template<>
struct __remove_unsigned<wchar_t>;
template<typename _Type>
constexpr
inline bool
__is_null_pointer(_Type* __ptr)
{ return __ptr == 0; }
template<typename _Type>
constexpr
inline bool
__is_null_pointer(_Type)
{ return false; }
constexpr bool
__is_null_pointer(std::nullptr_t)
{ return true; }
template<typename _Tp, bool = std::__is_integer<_Tp>::__value>
struct __promote
{ typedef double __type; };
template<typename _Tp>
struct __promote<_Tp, false>
{ };
template<>
struct __promote<long double>
{ typedef long double __type; };
template<>
struct __promote<double>
{ typedef double __type; };
template<>
struct __promote<float>
{ typedef float __type; };
template<typename... _Tp>
using __promoted_t = decltype((typename __promote<_Tp>::__type(0) + ...));
template<typename _Tp, typename _Up>
using __promote_2 = __promote<__promoted_t<_Tp, _Up>>;
template<typename _Tp, typename _Up, typename _Vp>
using __promote_3 = __promote<__promoted_t<_Tp, _Up, _Vp>>;
template<typename _Tp, typename _Up, typename _Vp, typename _Wp>
using __promote_4 = __promote<__promoted_t<_Tp, _Up, _Vp, _Wp>>;
}
}
extern "C" {
typedef long double float_t;
typedef long double double_t;
enum
{
FP_INT_UPWARD =
0,
FP_INT_DOWNWARD =
1,
FP_INT_TOWARDZERO =
2,
FP_INT_TONEARESTFROMZERO =
3,
FP_INT_TONEAREST =
4,
};
extern int __fpclassify (double __value) noexcept (true)
__attribute__ ((__const__));
extern int __signbit (double __value) noexcept (true)
__attribute__ ((__const__));
extern int __isinf (double __value) noexcept (true)
__attribute__ ((__const__));
extern int __finite (double __value) noexcept (true)
__attribute__ ((__const__));
extern int __isnan (double __value) noexcept (true)
__attribute__ ((__const__));
extern int __iseqsig (double __x, double __y) noexcept (true);
extern int __issignaling (double __value) noexcept (true)
__attribute__ ((__const__));
extern double acos (double __x) noexcept (true); extern double __acos (double __x) noexcept (true);
extern double asin (double __x) noexcept (true); extern double __asin (double __x) noexcept (true);
extern double atan (double __x) noexcept (true); extern double __atan (double __x) noexcept (true);
extern double atan2 (double __y, double __x) noexcept (true); extern double __atan2 (double __y, double __x) noexcept (true);
extern double cos (double __x) noexcept (true); extern double __cos (double __x) noexcept (true);
extern double sin (double __x) noexcept (true); extern double __sin (double __x) noexcept (true);
extern double tan (double __x) noexcept (true); extern double __tan (double __x) noexcept (true);
extern double cosh (double __x) noexcept (true); extern double __cosh (double __x) noexcept (true);
extern double sinh (double __x) noexcept (true); extern double __sinh (double __x) noexcept (true);
extern double tanh (double __x) noexcept (true); extern double __tanh (double __x) noexcept (true);
extern void sincos (double __x, double *__sinx, double *__cosx) noexcept (true); extern void __sincos (double __x, double *__sinx, double *__cosx) noexcept (true)
;
extern double acosh (double __x) noexcept (true); extern double __acosh (double __x) noexcept (true);
extern double asinh (double __x) noexcept (true); extern double __asinh (double __x) noexcept (true);
extern double atanh (double __x) noexcept (true); extern double __atanh (double __x) noexcept (true);
extern double exp (double __x) noexcept (true); extern double __exp (double __x) noexcept (true);
extern double frexp (double __x, int *__exponent) noexcept (true); extern double __frexp (double __x, int *__exponent) noexcept (true);
extern double ldexp (double __x, int __exponent) noexcept (true); extern double __ldexp (double __x, int __exponent) noexcept (true);
extern double log (double __x) noexcept (true); extern double __log (double __x) noexcept (true);
extern double log10 (double __x) noexcept (true); extern double __log10 (double __x) noexcept (true);
extern double modf (double __x, double *__iptr) noexcept (true); extern double __modf (double __x, double *__iptr) noexcept (true) __attribute__ ((__nonnull__ (2)));
extern double exp10 (double __x) noexcept (true); extern double __exp10 (double __x) noexcept (true);
extern double expm1 (double __x) noexcept (true); extern double __expm1 (double __x) noexcept (true);
extern double log1p (double __x) noexcept (true); extern double __log1p (double __x) noexcept (true);
extern double logb (double __x) noexcept (true); extern double __logb (double __x) noexcept (true);
extern double exp2 (double __x) noexcept (true); extern double __exp2 (double __x) noexcept (true);
extern double log2 (double __x) noexcept (true); extern double __log2 (double __x) noexcept (true);
extern double pow (double __x, double __y) noexcept (true); extern double __pow (double __x, double __y) noexcept (true);
extern double sqrt (double __x) noexcept (true); extern double __sqrt (double __x) noexcept (true);
extern double hypot (double __x, double __y) noexcept (true); extern double __hypot (double __x, double __y) noexcept (true);
extern double cbrt (double __x) noexcept (true); extern double __cbrt (double __x) noexcept (true);
extern double ceil (double __x) noexcept (true) __attribute__ ((__const__)); extern double __ceil (double __x) noexcept (true) __attribute__ ((__const__));
extern double fabs (double __x) noexcept (true) __attribute__ ((__const__)); extern double __fabs (double __x) noexcept (true) __attribute__ ((__const__));
extern double floor (double __x) noexcept (true) __attribute__ ((__const__)); extern double __floor (double __x) noexcept (true) __attribute__ ((__const__));
extern double fmod (double __x, double __y) noexcept (true); extern double __fmod (double __x, double __y) noexcept (true);
extern int finite (double __value) noexcept (true)
__attribute__ ((__const__));
extern double drem (double __x, double __y) noexcept (true); extern double __drem (double __x, double __y) noexcept (true);
extern double significand (double __x) noexcept (true); extern double __significand (double __x) noexcept (true);
extern double copysign (double __x, double __y) noexcept (true) __attribute__ ((__const__)); extern double __copysign (double __x, double __y) noexcept (true) __attribute__ ((__const__));
extern double nan (const char *__tagb) noexcept (true); extern double __nan (const char *__tagb) noexcept (true);
extern double j0 (double) noexcept (true); extern double __j0 (double) noexcept (true);
extern double j1 (double) noexcept (true); extern double __j1 (double) noexcept (true);
extern double jn (int, double) noexcept (true); extern double __jn (int, double) noexcept (true);
extern double y0 (double) noexcept (true); extern double __y0 (double) noexcept (true);
extern double y1 (double) noexcept (true); extern double __y1 (double) noexcept (true);
extern double yn (int, double) noexcept (true); extern double __yn (int, double) noexcept (true);
extern double erf (double) noexcept (true); extern double __erf (double) noexcept (true);
extern double erfc (double) noexcept (true); extern double __erfc (double) noexcept (true);
extern double lgamma (double) noexcept (true); extern double __lgamma (double) noexcept (true);
extern double tgamma (double) noexcept (true); extern double __tgamma (double) noexcept (true);
extern double gamma (double) noexcept (true); extern double __gamma (double) noexcept (true);
extern double lgamma_r (double, int *__signgamp) noexcept (true); extern double __lgamma_r (double, int *__signgamp) noexcept (true);
extern double rint (double __x) noexcept (true); extern double __rint (double __x) noexcept (true);
extern double nextafter (double __x, double __y) noexcept (true); extern double __nextafter (double __x, double __y) noexcept (true);
extern double nexttoward (double __x, long double __y) noexcept (true); extern double __nexttoward (double __x, long double __y) noexcept (true);
extern double nextdown (double __x) noexcept (true); extern double __nextdown (double __x) noexcept (true);
extern double nextup (double __x) noexcept (true); extern double __nextup (double __x) noexcept (true);
extern double remainder (double __x, double __y) noexcept (true); extern double __remainder (double __x, double __y) noexcept (true);
extern double scalbn (double __x, int __n) noexcept (true); extern double __scalbn (double __x, int __n) noexcept (true);
extern int ilogb (double __x) noexcept (true); extern int __ilogb (double __x) noexcept (true);
extern long int llogb (double __x) noexcept (true); extern long int __llogb (double __x) noexcept (true);
extern double scalbln (double __x, long int __n) noexcept (true); extern double __scalbln (double __x, long int __n) noexcept (true);
extern double nearbyint (double __x) noexcept (true); extern double __nearbyint (double __x) noexcept (true);
extern double round (double __x) noexcept (true) __attribute__ ((__const__)); extern double __round (double __x) noexcept (true) __attribute__ ((__const__));
extern double trunc (double __x) noexcept (true) __attribute__ ((__const__)); extern double __trunc (double __x) noexcept (true) __attribute__ ((__const__));
extern double remquo (double __x, double __y, int *__quo) noexcept (true); extern double __remquo (double __x, double __y, int *__quo) noexcept (true);
extern long int lrint (double __x) noexcept (true); extern long int __lrint (double __x) noexcept (true);
__extension__
extern long long int llrint (double __x) noexcept (true); extern long long int __llrint (double __x) noexcept (true);
extern long int lround (double __x) noexcept (true); extern long int __lround (double __x) noexcept (true);
__extension__
extern long long int llround (double __x) noexcept (true); extern long long int __llround (double __x) noexcept (true);
extern double fdim (double __x, double __y) noexcept (true); extern double __fdim (double __x, double __y) noexcept (true);
extern double fmax (double __x, double __y) noexcept (true) __attribute__ ((__const__)); extern double __fmax (double __x, double __y) noexcept (true) __attribute__ ((__const__));
extern double fmin (double __x, double __y) noexcept (true) __attribute__ ((__const__)); extern double __fmin (double __x, double __y) noexcept (true) __attribute__ ((__const__));
extern double fma (double __x, double __y, double __z) noexcept (true); extern double __fma (double __x, double __y, double __z) noexcept (true);
extern double roundeven (double __x) noexcept (true) __attribute__ ((__const__)); extern double __roundeven (double __x) noexcept (true) __attribute__ ((__const__));
extern __intmax_t fromfp (double __x, int __round, unsigned int __width) noexcept (true); extern __intmax_t __fromfp (double __x, int __round, unsigned int __width) noexcept (true)
;
extern __uintmax_t ufromfp (double __x, int __round, unsigned int __width) noexcept (true); extern __uintmax_t __ufromfp (double __x, int __round, unsigned int __width) noexcept (true)
;
extern __intmax_t fromfpx (double __x, int __round, unsigned int __width) noexcept (true); extern __intmax_t __fromfpx (double __x, int __round, unsigned int __width) noexcept (true)
;
extern __uintmax_t ufromfpx (double __x, int __round, unsigned int __width) noexcept (true); extern __uintmax_t __ufromfpx (double __x, int __round, unsigned int __width) noexcept (true)
;
extern int canonicalize (double *__cx, const double *__x) noexcept (true);
extern double fmaxmag (double __x, double __y) noexcept (true) __attribute__ ((__const__)); extern double __fmaxmag (double __x, double __y) noexcept (true) __attribute__ ((__const__));
extern double fminmag (double __x, double __y) noexcept (true) __attribute__ ((__const__)); extern double __fminmag (double __x, double __y) noexcept (true) __attribute__ ((__const__));
extern double fmaximum (double __x, double __y) noexcept (true) __attribute__ ((__const__)); extern double __fmaximum (double __x, double __y) noexcept (true) __attribute__ ((__const__));
extern double fminimum (double __x, double __y) noexcept (true) __attribute__ ((__const__)); extern double __fminimum (double __x, double __y) noexcept (true) __attribute__ ((__const__));
extern double fmaximum_num (double __x, double __y) noexcept (true) __attribute__ ((__const__)); extern double __fmaximum_num (double __x, double __y) noexcept (true) __attribute__ ((__const__));
extern double fminimum_num (double __x, double __y) noexcept (true) __attribute__ ((__const__)); extern double __fminimum_num (double __x, double __y) noexcept (true) __attribute__ ((__const__));
extern double fmaximum_mag (double __x, double __y) noexcept (true) __attribute__ ((__const__)); extern double __fmaximum_mag (double __x, double __y) noexcept (true) __attribute__ ((__const__));
extern double fminimum_mag (double __x, double __y) noexcept (true) __attribute__ ((__const__)); extern double __fminimum_mag (double __x, double __y) noexcept (true) __attribute__ ((__const__));
extern double fmaximum_mag_num (double __x, double __y) noexcept (true) __attribute__ ((__const__)); extern double __fmaximum_mag_num (double __x, double __y) noexcept (true) __attribute__ ((__const__));
extern double fminimum_mag_num (double __x, double __y) noexcept (true) __attribute__ ((__const__)); extern double __fminimum_mag_num (double __x, double __y) noexcept (true) __attribute__ ((__const__));
extern int totalorder (const double *__x, const double *__y) noexcept (true)
__attribute__ ((__pure__));
extern int totalordermag (const double *__x, const double *__y) noexcept (true)
__attribute__ ((__pure__));
extern double getpayload (const double *__x) noexcept (true); extern double __getpayload (const double *__x) noexcept (true);
extern int setpayload (double *__x, double __payload) noexcept (true);
extern int setpayloadsig (double *__x, double __payload) noexcept (true);
extern double scalb (double __x, double __n) noexcept (true); extern double __scalb (double __x, double __n) noexcept (true);
extern int __fpclassifyf (float __value) noexcept (true)
__attribute__ ((__const__));
extern int __signbitf (float __value) noexcept (true)
__attribute__ ((__const__));
extern int __isinff (float __value) noexcept (true)
__attribute__ ((__const__));
extern int __finitef (float __value) noexcept (true)
__attribute__ ((__const__));
extern int __isnanf (float __value) noexcept (true)
__attribute__ ((__const__));
extern int __iseqsigf (float __x, float __y) noexcept (true);
extern int __issignalingf (float __value) noexcept (true)
__attribute__ ((__const__));
extern float acosf (float __x) noexcept (true); extern float __acosf (float __x) noexcept (true);
extern float asinf (float __x) noexcept (true); extern float __asinf (float __x) noexcept (true);
extern float atanf (float __x) noexcept (true); extern float __atanf (float __x) noexcept (true);
extern float atan2f (float __y, float __x) noexcept (true); extern float __atan2f (float __y, float __x) noexcept (true);
extern float cosf (float __x) noexcept (true); extern float __cosf (float __x) noexcept (true);
extern float sinf (float __x) noexcept (true); extern float __sinf (float __x) noexcept (true);
extern float tanf (float __x) noexcept (true); extern float __tanf (float __x) noexcept (true);
extern float coshf (float __x) noexcept (true); extern float __coshf (float __x) noexcept (true);
extern float sinhf (float __x) noexcept (true); extern float __sinhf (float __x) noexcept (true);
extern float tanhf (float __x) noexcept (true); extern float __tanhf (float __x) noexcept (true);
extern void sincosf (float __x, float *__sinx, float *__cosx) noexcept (true); extern void __sincosf (float __x, float *__sinx, float *__cosx) noexcept (true)
;
extern float acoshf (float __x) noexcept (true); extern float __acoshf (float __x) noexcept (true);
extern float asinhf (float __x) noexcept (true); extern float __asinhf (float __x) noexcept (true);
extern float atanhf (float __x) noexcept (true); extern float __atanhf (float __x) noexcept (true);
extern float expf (float __x) noexcept (true); extern float __expf (float __x) noexcept (true);
extern float frexpf (float __x, int *__exponent) noexcept (true); extern float __frexpf (float __x, int *__exponent) noexcept (true);
extern float ldexpf (float __x, int __exponent) noexcept (true); extern float __ldexpf (float __x, int __exponent) noexcept (true);
extern float logf (float __x) noexcept (true); extern float __logf (float __x) noexcept (true);
extern float log10f (float __x) noexcept (true); extern float __log10f (float __x) noexcept (true);
extern float modff (float __x, float *__iptr) noexcept (true); extern float __modff (float __x, float *__iptr) noexcept (true) __attribute__ ((__nonnull__ (2)));
extern float exp10f (float __x) noexcept (true); extern float __exp10f (float __x) noexcept (true);
extern float expm1f (float __x) noexcept (true); extern float __expm1f (float __x) noexcept (true);
extern float log1pf (float __x) noexcept (true); extern float __log1pf (float __x) noexcept (true);
extern float logbf (float __x) noexcept (true); extern float __logbf (float __x) noexcept (true);
extern float exp2f (float __x) noexcept (true); extern float __exp2f (float __x) noexcept (true);
extern float log2f (float __x) noexcept (true); extern float __log2f (float __x) noexcept (true);
extern float powf (float __x, float __y) noexcept (true); extern float __powf (float __x, float __y) noexcept (true);
extern float sqrtf (float __x) noexcept (true); extern float __sqrtf (float __x) noexcept (true);
extern float hypotf (float __x, float __y) noexcept (true); extern float __hypotf (float __x, float __y) noexcept (true);
extern float cbrtf (float __x) noexcept (true); extern float __cbrtf (float __x) noexcept (true);
extern float ceilf (float __x) noexcept (true) __attribute__ ((__const__)); extern float __ceilf (float __x) noexcept (true) __attribute__ ((__const__));
extern float fabsf (float __x) noexcept (true) __attribute__ ((__const__)); extern float __fabsf (float __x) noexcept (true) __attribute__ ((__const__));
extern float floorf (float __x) noexcept (true) __attribute__ ((__const__)); extern float __floorf (float __x) noexcept (true) __attribute__ ((__const__));
extern float fmodf (float __x, float __y) noexcept (true); extern float __fmodf (float __x, float __y) noexcept (true);
extern int isinff (float __value) noexcept (true)
__attribute__ ((__const__));
extern int finitef (float __value) noexcept (true)
__attribute__ ((__const__));
extern float dremf (float __x, float __y) noexcept (true); extern float __dremf (float __x, float __y) noexcept (true);
extern float significandf (float __x) noexcept (true); extern float __significandf (float __x) noexcept (true);
extern float copysignf (float __x, float __y) noexcept (true) __attribute__ ((__const__)); extern float __copysignf (float __x, float __y) noexcept (true) __attribute__ ((__const__));
extern float nanf (const char *__tagb) noexcept (true); extern float __nanf (const char *__tagb) noexcept (true);
extern int isnanf (float __value) noexcept (true)
__attribute__ ((__const__));
extern float j0f (float) noexcept (true); extern float __j0f (float) noexcept (true);
extern float j1f (float) noexcept (true); extern float __j1f (float) noexcept (true);
extern float jnf (int, float) noexcept (true); extern float __jnf (int, float) noexcept (true);
extern float y0f (float) noexcept (true); extern float __y0f (float) noexcept (true);
extern float y1f (float) noexcept (true); extern float __y1f (float) noexcept (true);
extern float ynf (int, float) noexcept (true); extern float __ynf (int, float) noexcept (true);
extern float erff (float) noexcept (true); extern float __erff (float) noexcept (true);
extern float erfcf (float) noexcept (true); extern float __erfcf (float) noexcept (true);
extern float lgammaf (float) noexcept (true); extern float __lgammaf (float) noexcept (true);
extern float tgammaf (float) noexcept (true); extern float __tgammaf (float) noexcept (true);
extern float gammaf (float) noexcept (true); extern float __gammaf (float) noexcept (true);
extern float lgammaf_r (float, int *__signgamp) noexcept (true); extern float __lgammaf_r (float, int *__signgamp) noexcept (true);
extern float rintf (float __x) noexcept (true); extern float __rintf (float __x) noexcept (true);
extern float nextafterf (float __x, float __y) noexcept (true); extern float __nextafterf (float __x, float __y) noexcept (true);
extern float nexttowardf (float __x, long double __y) noexcept (true); extern float __nexttowardf (float __x, long double __y) noexcept (true);
extern float nextdownf (float __x) noexcept (true); extern float __nextdownf (float __x) noexcept (true);
extern float nextupf (float __x) noexcept (true); extern float __nextupf (float __x) noexcept (true);
extern float remainderf (float __x, float __y) noexcept (true); extern float __remainderf (float __x, float __y) noexcept (true);
extern float scalbnf (float __x, int __n) noexcept (true); extern float __scalbnf (float __x, int __n) noexcept (true);
extern int ilogbf (float __x) noexcept (true); extern int __ilogbf (float __x) noexcept (true);
extern long int llogbf (float __x) noexcept (true); extern long int __llogbf (float __x) noexcept (true);
extern float scalblnf (float __x, long int __n) noexcept (true); extern float __scalblnf (float __x, long int __n) noexcept (true);
extern float nearbyintf (float __x) noexcept (true); extern float __nearbyintf (float __x) noexcept (true);
extern float roundf (float __x) noexcept (true) __attribute__ ((__const__)); extern float __roundf (float __x) noexcept (true) __attribute__ ((__const__));
extern float truncf (float __x) noexcept (true) __attribute__ ((__const__)); extern float __truncf (float __x) noexcept (true) __attribute__ ((__const__));
extern float remquof (float __x, float __y, int *__quo) noexcept (true); extern float __remquof (float __x, float __y, int *__quo) noexcept (true);
extern long int lrintf (float __x) noexcept (true); extern long int __lrintf (float __x) noexcept (true);
__extension__
extern long long int llrintf (float __x) noexcept (true); extern long long int __llrintf (float __x) noexcept (true);
extern long int lroundf (float __x) noexcept (true); extern long int __lroundf (float __x) noexcept (true);
__extension__
extern long long int llroundf (float __x) noexcept (true); extern long long int __llroundf (float __x) noexcept (true);
extern float fdimf (float __x, float __y) noexcept (true); extern float __fdimf (float __x, float __y) noexcept (true);
extern float fmaxf (float __x, float __y) noexcept (true) __attribute__ ((__const__)); extern float __fmaxf (float __x, float __y) noexcept (true) __attribute__ ((__const__));
extern float fminf (float __x, float __y) noexcept (true) __attribute__ ((__const__)); extern float __fminf (float __x, float __y) noexcept (true) __attribute__ ((__const__));
extern float fmaf (float __x, float __y, float __z) noexcept (true); extern float __fmaf (float __x, float __y, float __z) noexcept (true);
extern float roundevenf (float __x) noexcept (true) __attribute__ ((__const__)); extern float __roundevenf (float __x) noexcept (true) __attribute__ ((__const__));
extern __intmax_t fromfpf (float __x, int __round, unsigned int __width) noexcept (true); extern __intmax_t __fromfpf (float __x, int __round, unsigned int __width) noexcept (true)
;
extern __uintmax_t ufromfpf (float __x, int __round, unsigned int __width) noexcept (true); extern __uintmax_t __ufromfpf (float __x, int __round, unsigned int __width) noexcept (true)
;
extern __intmax_t fromfpxf (float __x, int __round, unsigned int __width) noexcept (true); extern __intmax_t __fromfpxf (float __x, int __round, unsigned int __width) noexcept (true)
;
extern __uintmax_t ufromfpxf (float __x, int __round, unsigned int __width) noexcept (true); extern __uintmax_t __ufromfpxf (float __x, int __round, unsigned int __width) noexcept (true)
;
extern int canonicalizef (float *__cx, const float *__x) noexcept (true);
extern float fmaxmagf (float __x, float __y) noexcept (true) __attribute__ ((__const__)); extern float __fmaxmagf (float __x, float __y) noexcept (true) __attribute__ ((__const__));
extern float fminmagf (float __x, float __y) noexcept (true) __attribute__ ((__const__)); extern float __fminmagf (float __x, float __y) noexcept (true) __attribute__ ((__const__));
extern float fmaximumf (float __x, float __y) noexcept (true) __attribute__ ((__const__)); extern float __fmaximumf (float __x, float __y) noexcept (true) __attribute__ ((__const__));
extern float fminimumf (float __x, float __y) noexcept (true) __attribute__ ((__const__)); extern float __fminimumf (float __x, float __y) noexcept (true) __attribute__ ((__const__));
extern float fmaximum_numf (float __x, float __y) noexcept (true) __attribute__ ((__const__)); extern float __fmaximum_numf (float __x, float __y) noexcept (true) __attribute__ ((__const__));
extern float fminimum_numf (float __x, float __y) noexcept (true) __attribute__ ((__const__)); extern float __fminimum_numf (float __x, float __y) noexcept (true) __attribute__ ((__const__));
extern float fmaximum_magf (float __x, float __y) noexcept (true) __attribute__ ((__const__)); extern float __fmaximum_magf (float __x, float __y) noexcept (true) __attribute__ ((__const__));
extern float fminimum_magf (float __x, float __y) noexcept (true) __attribute__ ((__const__)); extern float __fminimum_magf (float __x, float __y) noexcept (true) __attribute__ ((__const__));
extern float fmaximum_mag_numf (float __x, float __y) noexcept (true) __attribute__ ((__const__)); extern float __fmaximum_mag_numf (float __x, float __y) noexcept (true) __attribute__ ((__const__));
extern float fminimum_mag_numf (float __x, float __y) noexcept (true) __attribute__ ((__const__)); extern float __fminimum_mag_numf (float __x, float __y) noexcept (true) __attribute__ ((__const__));
extern int totalorderf (const float *__x, const float *__y) noexcept (true)
__attribute__ ((__pure__));
extern int totalordermagf (const float *__x, const float *__y) noexcept (true)
__attribute__ ((__pure__));
extern float getpayloadf (const float *__x) noexcept (true); extern float __getpayloadf (const float *__x) noexcept (true);
extern int setpayloadf (float *__x, float __payload) noexcept (true);
extern int setpayloadsigf (float *__x, float __payload) noexcept (true);
extern float scalbf (float __x, float __n) noexcept (true); extern float __scalbf (float __x, float __n) noexcept (true);
extern int __fpclassifyl (long double __value) noexcept (true)
__attribute__ ((__const__));
extern int __signbitl (long double __value) noexcept (true)
__attribute__ ((__const__));
extern int __isinfl (long double __value) noexcept (true)
__attribute__ ((__const__));
extern int __finitel (long double __value) noexcept (true)
__attribute__ ((__const__));
extern int __isnanl (long double __value) noexcept (true)
__attribute__ ((__const__));
extern int __iseqsigl (long double __x, long double __y) noexcept (true);
extern int __issignalingl (long double __value) noexcept (true)
__attribute__ ((__const__));
extern long double acosl (long double __x) noexcept (true); extern long double __acosl (long double __x) noexcept (true);
extern long double asinl (long double __x) noexcept (true); extern long double __asinl (long double __x) noexcept (true);
extern long double atanl (long double __x) noexcept (true); extern long double __atanl (long double __x) noexcept (true);
extern long double atan2l (long double __y, long double __x) noexcept (true); extern long double __atan2l (long double __y, long double __x) noexcept (true);
extern long double cosl (long double __x) noexcept (true); extern long double __cosl (long double __x) noexcept (true);
extern long double sinl (long double __x) noexcept (true); extern long double __sinl (long double __x) noexcept (true);
extern long double tanl (long double __x) noexcept (true); extern long double __tanl (long double __x) noexcept (true);
extern long double coshl (long double __x) noexcept (true); extern long double __coshl (long double __x) noexcept (true);
extern long double sinhl (long double __x) noexcept (true); extern long double __sinhl (long double __x) noexcept (true);
extern long double tanhl (long double __x) noexcept (true); extern long double __tanhl (long double __x) noexcept (true);
extern void sincosl (long double __x, long double *__sinx, long double *__cosx) noexcept (true); extern void __sincosl (long double __x, long double *__sinx, long double *__cosx) noexcept (true)
;
extern long double acoshl (long double __x) noexcept (true); extern long double __acoshl (long double __x) noexcept (true);
extern long double asinhl (long double __x) noexcept (true); extern long double __asinhl (long double __x) noexcept (true);
extern long double atanhl (long double __x) noexcept (true); extern long double __atanhl (long double __x) noexcept (true);
extern long double expl (long double __x) noexcept (true); extern long double __expl (long double __x) noexcept (true);
extern long double frexpl (long double __x, int *__exponent) noexcept (true); extern long double __frexpl (long double __x, int *__exponent) noexcept (true);
extern long double ldexpl (long double __x, int __exponent) noexcept (true); extern long double __ldexpl (long double __x, int __exponent) noexcept (true);
extern long double logl (long double __x) noexcept (true); extern long double __logl (long double __x) noexcept (true);
extern long double log10l (long double __x) noexcept (true); extern long double __log10l (long double __x) noexcept (true);
extern long double modfl (long double __x, long double *__iptr) noexcept (true); extern long double __modfl (long double __x, long double *__iptr) noexcept (true) __attribute__ ((__nonnull__ (2)));
extern long double exp10l (long double __x) noexcept (true); extern long double __exp10l (long double __x) noexcept (true);
extern long double expm1l (long double __x) noexcept (true); extern long double __expm1l (long double __x) noexcept (true);
extern long double log1pl (long double __x) noexcept (true); extern long double __log1pl (long double __x) noexcept (true);
extern long double logbl (long double __x) noexcept (true); extern long double __logbl (long double __x) noexcept (true);
extern long double exp2l (long double __x) noexcept (true); extern long double __exp2l (long double __x) noexcept (true);
extern long double log2l (long double __x) noexcept (true); extern long double __log2l (long double __x) noexcept (true);
extern long double powl (long double __x, long double __y) noexcept (true); extern long double __powl (long double __x, long double __y) noexcept (true);
extern long double sqrtl (long double __x) noexcept (true); extern long double __sqrtl (long double __x) noexcept (true);
extern long double hypotl (long double __x, long double __y) noexcept (true); extern long double __hypotl (long double __x, long double __y) noexcept (true);
extern long double cbrtl (long double __x) noexcept (true); extern long double __cbrtl (long double __x) noexcept (true);
extern long double ceill (long double __x) noexcept (true) __attribute__ ((__const__)); extern long double __ceill (long double __x) noexcept (true) __attribute__ ((__const__));
extern long double fabsl (long double __x) noexcept (true) __attribute__ ((__const__)); extern long double __fabsl (long double __x) noexcept (true) __attribute__ ((__const__));
extern long double floorl (long double __x) noexcept (true) __attribute__ ((__const__)); extern long double __floorl (long double __x) noexcept (true) __attribute__ ((__const__));
extern long double fmodl (long double __x, long double __y) noexcept (true); extern long double __fmodl (long double __x, long double __y) noexcept (true);
extern int isinfl (long double __value) noexcept (true)
__attribute__ ((__const__));
extern int finitel (long double __value) noexcept (true)
__attribute__ ((__const__));
extern long double dreml (long double __x, long double __y) noexcept (true); extern long double __dreml (long double __x, long double __y) noexcept (true);
extern long double significandl (long double __x) noexcept (true); extern long double __significandl (long double __x) noexcept (true);
extern long double copysignl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__)); extern long double __copysignl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__));
extern long double nanl (const char *__tagb) noexcept (true); extern long double __nanl (const char *__tagb) noexcept (true);
extern int isnanl (long double __value) noexcept (true)
__attribute__ ((__const__));
extern long double j0l (long double) noexcept (true); extern long double __j0l (long double) noexcept (true);
extern long double j1l (long double) noexcept (true); extern long double __j1l (long double) noexcept (true);
extern long double jnl (int, long double) noexcept (true); extern long double __jnl (int, long double) noexcept (true);
extern long double y0l (long double) noexcept (true); extern long double __y0l (long double) noexcept (true);
extern long double y1l (long double) noexcept (true); extern long double __y1l (long double) noexcept (true);
extern long double ynl (int, long double) noexcept (true); extern long double __ynl (int, long double) noexcept (true);
extern long double erfl (long double) noexcept (true); extern long double __erfl (long double) noexcept (true);
extern long double erfcl (long double) noexcept (true); extern long double __erfcl (long double) noexcept (true);
extern long double lgammal (long double) noexcept (true); extern long double __lgammal (long double) noexcept (true);
extern long double tgammal (long double) noexcept (true); extern long double __tgammal (long double) noexcept (true);
extern long double gammal (long double) noexcept (true); extern long double __gammal (long double) noexcept (true);
extern long double lgammal_r (long double, int *__signgamp) noexcept (true); extern long double __lgammal_r (long double, int *__signgamp) noexcept (true);
extern long double rintl (long double __x) noexcept (true); extern long double __rintl (long double __x) noexcept (true);
extern long double nextafterl (long double __x, long double __y) noexcept (true); extern long double __nextafterl (long double __x, long double __y) noexcept (true);
extern long double nexttowardl (long double __x, long double __y) noexcept (true); extern long double __nexttowardl (long double __x, long double __y) noexcept (true);
extern long double nextdownl (long double __x) noexcept (true); extern long double __nextdownl (long double __x) noexcept (true);
extern long double nextupl (long double __x) noexcept (true); extern long double __nextupl (long double __x) noexcept (true);
extern long double remainderl (long double __x, long double __y) noexcept (true); extern long double __remainderl (long double __x, long double __y) noexcept (true);
extern long double scalbnl (long double __x, int __n) noexcept (true); extern long double __scalbnl (long double __x, int __n) noexcept (true);
extern int ilogbl (long double __x) noexcept (true); extern int __ilogbl (long double __x) noexcept (true);
extern long int llogbl (long double __x) noexcept (true); extern long int __llogbl (long double __x) noexcept (true);
extern long double scalblnl (long double __x, long int __n) noexcept (true); extern long double __scalblnl (long double __x, long int __n) noexcept (true);
extern long double nearbyintl (long double __x) noexcept (true); extern long double __nearbyintl (long double __x) noexcept (true);
extern long double roundl (long double __x) noexcept (true) __attribute__ ((__const__)); extern long double __roundl (long double __x) noexcept (true) __attribute__ ((__const__));
extern long double truncl (long double __x) noexcept (true) __attribute__ ((__const__)); extern long double __truncl (long double __x) noexcept (true) __attribute__ ((__const__));
extern long double remquol (long double __x, long double __y, int *__quo) noexcept (true); extern long double __remquol (long double __x, long double __y, int *__quo) noexcept (true);
extern long int lrintl (long double __x) noexcept (true); extern long int __lrintl (long double __x) noexcept (true);
__extension__
extern long long int llrintl (long double __x) noexcept (true); extern long long int __llrintl (long double __x) noexcept (true);
extern long int lroundl (long double __x) noexcept (true); extern long int __lroundl (long double __x) noexcept (true);
__extension__
extern long long int llroundl (long double __x) noexcept (true); extern long long int __llroundl (long double __x) noexcept (true);
extern long double fdiml (long double __x, long double __y) noexcept (true); extern long double __fdiml (long double __x, long double __y) noexcept (true);
extern long double fmaxl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__)); extern long double __fmaxl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__));
extern long double fminl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__)); extern long double __fminl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__));
extern long double fmal (long double __x, long double __y, long double __z) noexcept (true); extern long double __fmal (long double __x, long double __y, long double __z) noexcept (true);
extern long double roundevenl (long double __x) noexcept (true) __attribute__ ((__const__)); extern long double __roundevenl (long double __x) noexcept (true) __attribute__ ((__const__));
extern __intmax_t fromfpl (long double __x, int __round, unsigned int __width) noexcept (true); extern __intmax_t __fromfpl (long double __x, int __round, unsigned int __width) noexcept (true)
;
extern __uintmax_t ufromfpl (long double __x, int __round, unsigned int __width) noexcept (true); extern __uintmax_t __ufromfpl (long double __x, int __round, unsigned int __width) noexcept (true)
;
extern __intmax_t fromfpxl (long double __x, int __round, unsigned int __width) noexcept (true); extern __intmax_t __fromfpxl (long double __x, int __round, unsigned int __width) noexcept (true)
;
extern __uintmax_t ufromfpxl (long double __x, int __round, unsigned int __width) noexcept (true); extern __uintmax_t __ufromfpxl (long double __x, int __round, unsigned int __width) noexcept (true)
;
extern int canonicalizel (long double *__cx, const long double *__x) noexcept (true);
extern long double fmaxmagl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__)); extern long double __fmaxmagl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__));
extern long double fminmagl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__)); extern long double __fminmagl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__));
extern long double fmaximuml (long double __x, long double __y) noexcept (true) __attribute__ ((__const__)); extern long double __fmaximuml (long double __x, long double __y) noexcept (true) __attribute__ ((__const__));
extern long double fminimuml (long double __x, long double __y) noexcept (true) __attribute__ ((__const__)); extern long double __fminimuml (long double __x, long double __y) noexcept (true) __attribute__ ((__const__));
extern long double fmaximum_numl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__)); extern long double __fmaximum_numl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__));
extern long double fminimum_numl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__)); extern long double __fminimum_numl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__));
extern long double fmaximum_magl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__)); extern long double __fmaximum_magl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__));
extern long double fminimum_magl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__)); extern long double __fminimum_magl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__));
extern long double fmaximum_mag_numl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__)); extern long double __fmaximum_mag_numl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__));
extern long double fminimum_mag_numl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__)); extern long double __fminimum_mag_numl (long double __x, long double __y) noexcept (true) __attribute__ ((__const__));
extern int totalorderl (const long double *__x, const long double *__y) noexcept (true)
__attribute__ ((__pure__));
extern int totalordermagl (const long double *__x, const long double *__y) noexcept (true)
__attribute__ ((__pure__));
extern long double getpayloadl (const long double *__x) noexcept (true); extern long double __getpayloadl (const long double *__x) noexcept (true);
extern int setpayloadl (long double *__x, long double __payload) noexcept (true);
extern int setpayloadsigl (long double *__x, long double __payload) noexcept (true);
extern long double scalbl (long double __x, long double __n) noexcept (true); extern long double __scalbl (long double __x, long double __n) noexcept (true);
extern _Float32 acosf32 (_Float32 __x) noexcept (true); extern _Float32 __acosf32 (_Float32 __x) noexcept (true);
extern _Float32 asinf32 (_Float32 __x) noexcept (true); extern _Float32 __asinf32 (_Float32 __x) noexcept (true);
extern _Float32 atanf32 (_Float32 __x) noexcept (true); extern _Float32 __atanf32 (_Float32 __x) noexcept (true);
extern _Float32 atan2f32 (_Float32 __y, _Float32 __x) noexcept (true); extern _Float32 __atan2f32 (_Float32 __y, _Float32 __x) noexcept (true);
extern _Float32 cosf32 (_Float32 __x) noexcept (true); extern _Float32 __cosf32 (_Float32 __x) noexcept (true);
extern _Float32 sinf32 (_Float32 __x) noexcept (true); extern _Float32 __sinf32 (_Float32 __x) noexcept (true);
extern _Float32 tanf32 (_Float32 __x) noexcept (true); extern _Float32 __tanf32 (_Float32 __x) noexcept (true);
extern _Float32 coshf32 (_Float32 __x) noexcept (true); extern _Float32 __coshf32 (_Float32 __x) noexcept (true);
extern _Float32 sinhf32 (_Float32 __x) noexcept (true); extern _Float32 __sinhf32 (_Float32 __x) noexcept (true);
extern _Float32 tanhf32 (_Float32 __x) noexcept (true); extern _Float32 __tanhf32 (_Float32 __x) noexcept (true);
extern void sincosf32 (_Float32 __x, _Float32 *__sinx, _Float32 *__cosx) noexcept (true); extern void __sincosf32 (_Float32 __x, _Float32 *__sinx, _Float32 *__cosx) noexcept (true)
;
extern _Float32 acoshf32 (_Float32 __x) noexcept (true); extern _Float32 __acoshf32 (_Float32 __x) noexcept (true);
extern _Float32 asinhf32 (_Float32 __x) noexcept (true); extern _Float32 __asinhf32 (_Float32 __x) noexcept (true);
extern _Float32 atanhf32 (_Float32 __x) noexcept (true); extern _Float32 __atanhf32 (_Float32 __x) noexcept (true);
extern _Float32 expf32 (_Float32 __x) noexcept (true); extern _Float32 __expf32 (_Float32 __x) noexcept (true);
extern _Float32 frexpf32 (_Float32 __x, int *__exponent) noexcept (true); extern _Float32 __frexpf32 (_Float32 __x, int *__exponent) noexcept (true);
extern _Float32 ldexpf32 (_Float32 __x, int __exponent) noexcept (true); extern _Float32 __ldexpf32 (_Float32 __x, int __exponent) noexcept (true);
extern _Float32 logf32 (_Float32 __x) noexcept (true); extern _Float32 __logf32 (_Float32 __x) noexcept (true);
extern _Float32 log10f32 (_Float32 __x) noexcept (true); extern _Float32 __log10f32 (_Float32 __x) noexcept (true);
extern _Float32 modff32 (_Float32 __x, _Float32 *__iptr) noexcept (true); extern _Float32 __modff32 (_Float32 __x, _Float32 *__iptr) noexcept (true) __attribute__ ((__nonnull__ (2)));
extern _Float32 exp10f32 (_Float32 __x) noexcept (true); extern _Float32 __exp10f32 (_Float32 __x) noexcept (true);
extern _Float32 expm1f32 (_Float32 __x) noexcept (true); extern _Float32 __expm1f32 (_Float32 __x) noexcept (true);
extern _Float32 log1pf32 (_Float32 __x) noexcept (true); extern _Float32 __log1pf32 (_Float32 __x) noexcept (true);
extern _Float32 logbf32 (_Float32 __x) noexcept (true); extern _Float32 __logbf32 (_Float32 __x) noexcept (true);
extern _Float32 exp2f32 (_Float32 __x) noexcept (true); extern _Float32 __exp2f32 (_Float32 __x) noexcept (true);
extern _Float32 log2f32 (_Float32 __x) noexcept (true); extern _Float32 __log2f32 (_Float32 __x) noexcept (true);
extern _Float32 powf32 (_Float32 __x, _Float32 __y) noexcept (true); extern _Float32 __powf32 (_Float32 __x, _Float32 __y) noexcept (true);
extern _Float32 sqrtf32 (_Float32 __x) noexcept (true); extern _Float32 __sqrtf32 (_Float32 __x) noexcept (true);
extern _Float32 hypotf32 (_Float32 __x, _Float32 __y) noexcept (true); extern _Float32 __hypotf32 (_Float32 __x, _Float32 __y) noexcept (true);
extern _Float32 cbrtf32 (_Float32 __x) noexcept (true); extern _Float32 __cbrtf32 (_Float32 __x) noexcept (true);
extern _Float32 ceilf32 (_Float32 __x) noexcept (true) __attribute__ ((__const__)); extern _Float32 __ceilf32 (_Float32 __x) noexcept (true) __attribute__ ((__const__));
extern _Float32 fabsf32 (_Float32 __x) noexcept (true) __attribute__ ((__const__)); extern _Float32 __fabsf32 (_Float32 __x) noexcept (true) __attribute__ ((__const__));
extern _Float32 floorf32 (_Float32 __x) noexcept (true) __attribute__ ((__const__)); extern _Float32 __floorf32 (_Float32 __x) noexcept (true) __attribute__ ((__const__));
extern _Float32 fmodf32 (_Float32 __x, _Float32 __y) noexcept (true); extern _Float32 __fmodf32 (_Float32 __x, _Float32 __y) noexcept (true);
extern _Float32 copysignf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__)); extern _Float32 __copysignf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__));
extern _Float32 nanf32 (const char *__tagb) noexcept (true); extern _Float32 __nanf32 (const char *__tagb) noexcept (true);
extern _Float32 j0f32 (_Float32) noexcept (true); extern _Float32 __j0f32 (_Float32) noexcept (true);
extern _Float32 j1f32 (_Float32) noexcept (true); extern _Float32 __j1f32 (_Float32) noexcept (true);
extern _Float32 jnf32 (int, _Float32) noexcept (true); extern _Float32 __jnf32 (int, _Float32) noexcept (true);
extern _Float32 y0f32 (_Float32) noexcept (true); extern _Float32 __y0f32 (_Float32) noexcept (true);
extern _Float32 y1f32 (_Float32) noexcept (true); extern _Float32 __y1f32 (_Float32) noexcept (true);
extern _Float32 ynf32 (int, _Float32) noexcept (true); extern _Float32 __ynf32 (int, _Float32) noexcept (true);
extern _Float32 erff32 (_Float32) noexcept (true); extern _Float32 __erff32 (_Float32) noexcept (true);
extern _Float32 erfcf32 (_Float32) noexcept (true); extern _Float32 __erfcf32 (_Float32) noexcept (true);
extern _Float32 lgammaf32 (_Float32) noexcept (true); extern _Float32 __lgammaf32 (_Float32) noexcept (true);
extern _Float32 tgammaf32 (_Float32) noexcept (true); extern _Float32 __tgammaf32 (_Float32) noexcept (true);
extern _Float32 lgammaf32_r (_Float32, int *__signgamp) noexcept (true); extern _Float32 __lgammaf32_r (_Float32, int *__signgamp) noexcept (true);
extern _Float32 rintf32 (_Float32 __x) noexcept (true); extern _Float32 __rintf32 (_Float32 __x) noexcept (true);
extern _Float32 nextafterf32 (_Float32 __x, _Float32 __y) noexcept (true); extern _Float32 __nextafterf32 (_Float32 __x, _Float32 __y) noexcept (true);
extern _Float32 nextdownf32 (_Float32 __x) noexcept (true); extern _Float32 __nextdownf32 (_Float32 __x) noexcept (true);
extern _Float32 nextupf32 (_Float32 __x) noexcept (true); extern _Float32 __nextupf32 (_Float32 __x) noexcept (true);
extern _Float32 remainderf32 (_Float32 __x, _Float32 __y) noexcept (true); extern _Float32 __remainderf32 (_Float32 __x, _Float32 __y) noexcept (true);
extern _Float32 scalbnf32 (_Float32 __x, int __n) noexcept (true); extern _Float32 __scalbnf32 (_Float32 __x, int __n) noexcept (true);
extern int ilogbf32 (_Float32 __x) noexcept (true); extern int __ilogbf32 (_Float32 __x) noexcept (true);
extern long int llogbf32 (_Float32 __x) noexcept (true); extern long int __llogbf32 (_Float32 __x) noexcept (true);
extern _Float32 scalblnf32 (_Float32 __x, long int __n) noexcept (true); extern _Float32 __scalblnf32 (_Float32 __x, long int __n) noexcept (true);
extern _Float32 nearbyintf32 (_Float32 __x) noexcept (true); extern _Float32 __nearbyintf32 (_Float32 __x) noexcept (true);
extern _Float32 roundf32 (_Float32 __x) noexcept (true) __attribute__ ((__const__)); extern _Float32 __roundf32 (_Float32 __x) noexcept (true) __attribute__ ((__const__));
extern _Float32 truncf32 (_Float32 __x) noexcept (true) __attribute__ ((__const__)); extern _Float32 __truncf32 (_Float32 __x) noexcept (true) __attribute__ ((__const__));
extern _Float32 remquof32 (_Float32 __x, _Float32 __y, int *__quo) noexcept (true); extern _Float32 __remquof32 (_Float32 __x, _Float32 __y, int *__quo) noexcept (true);
extern long int lrintf32 (_Float32 __x) noexcept (true); extern long int __lrintf32 (_Float32 __x) noexcept (true);
__extension__
extern long long int llrintf32 (_Float32 __x) noexcept (true); extern long long int __llrintf32 (_Float32 __x) noexcept (true);
extern long int lroundf32 (_Float32 __x) noexcept (true); extern long int __lroundf32 (_Float32 __x) noexcept (true);
__extension__
extern long long int llroundf32 (_Float32 __x) noexcept (true); extern long long int __llroundf32 (_Float32 __x) noexcept (true);
extern _Float32 fdimf32 (_Float32 __x, _Float32 __y) noexcept (true); extern _Float32 __fdimf32 (_Float32 __x, _Float32 __y) noexcept (true);
extern _Float32 fmaxf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__)); extern _Float32 __fmaxf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__));
extern _Float32 fminf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__)); extern _Float32 __fminf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__));
extern _Float32 fmaf32 (_Float32 __x, _Float32 __y, _Float32 __z) noexcept (true); extern _Float32 __fmaf32 (_Float32 __x, _Float32 __y, _Float32 __z) noexcept (true);
extern _Float32 roundevenf32 (_Float32 __x) noexcept (true) __attribute__ ((__const__)); extern _Float32 __roundevenf32 (_Float32 __x) noexcept (true) __attribute__ ((__const__));
extern __intmax_t fromfpf32 (_Float32 __x, int __round, unsigned int __width) noexcept (true); extern __intmax_t __fromfpf32 (_Float32 __x, int __round, unsigned int __width) noexcept (true)
;
extern __uintmax_t ufromfpf32 (_Float32 __x, int __round, unsigned int __width) noexcept (true); extern __uintmax_t __ufromfpf32 (_Float32 __x, int __round, unsigned int __width) noexcept (true)
;
extern __intmax_t fromfpxf32 (_Float32 __x, int __round, unsigned int __width) noexcept (true); extern __intmax_t __fromfpxf32 (_Float32 __x, int __round, unsigned int __width) noexcept (true)
;
extern __uintmax_t ufromfpxf32 (_Float32 __x, int __round, unsigned int __width) noexcept (true); extern __uintmax_t __ufromfpxf32 (_Float32 __x, int __round, unsigned int __width) noexcept (true)
;
extern int canonicalizef32 (_Float32 *__cx, const _Float32 *__x) noexcept (true);
extern _Float32 fmaxmagf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__)); extern _Float32 __fmaxmagf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__));
extern _Float32 fminmagf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__)); extern _Float32 __fminmagf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__));
extern _Float32 fmaximumf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__)); extern _Float32 __fmaximumf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__));
extern _Float32 fminimumf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__)); extern _Float32 __fminimumf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__));
extern _Float32 fmaximum_numf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__)); extern _Float32 __fmaximum_numf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__));
extern _Float32 fminimum_numf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__)); extern _Float32 __fminimum_numf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__));
extern _Float32 fmaximum_magf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__)); extern _Float32 __fmaximum_magf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__));
extern _Float32 fminimum_magf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__)); extern _Float32 __fminimum_magf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__));
extern _Float32 fmaximum_mag_numf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__)); extern _Float32 __fmaximum_mag_numf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__));
extern _Float32 fminimum_mag_numf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__)); extern _Float32 __fminimum_mag_numf32 (_Float32 __x, _Float32 __y) noexcept (true) __attribute__ ((__const__));
extern int totalorderf32 (const _Float32 *__x, const _Float32 *__y) noexcept (true)
__attribute__ ((__pure__));
extern int totalordermagf32 (const _Float32 *__x, const _Float32 *__y) noexcept (true)
__attribute__ ((__pure__));
extern _Float32 getpayloadf32 (const _Float32 *__x) noexcept (true); extern _Float32 __getpayloadf32 (const _Float32 *__x) noexcept (true);
extern int setpayloadf32 (_Float32 *__x, _Float32 __payload) noexcept (true);
extern int setpayloadsigf32 (_Float32 *__x, _Float32 __payload) noexcept (true);
extern _Float64 acosf64 (_Float64 __x) noexcept (true); extern _Float64 __acosf64 (_Float64 __x) noexcept (true);
extern _Float64 asinf64 (_Float64 __x) noexcept (true); extern _Float64 __asinf64 (_Float64 __x) noexcept (true);
extern _Float64 atanf64 (_Float64 __x) noexcept (true); extern _Float64 __atanf64 (_Float64 __x) noexcept (true);
extern _Float64 atan2f64 (_Float64 __y, _Float64 __x) noexcept (true); extern _Float64 __atan2f64 (_Float64 __y, _Float64 __x) noexcept (true);
extern _Float64 cosf64 (_Float64 __x) noexcept (true); extern _Float64 __cosf64 (_Float64 __x) noexcept (true);
extern _Float64 sinf64 (_Float64 __x) noexcept (true); extern _Float64 __sinf64 (_Float64 __x) noexcept (true);
extern _Float64 tanf64 (_Float64 __x) noexcept (true); extern _Float64 __tanf64 (_Float64 __x) noexcept (true);
extern _Float64 coshf64 (_Float64 __x) noexcept (true); extern _Float64 __coshf64 (_Float64 __x) noexcept (true);
extern _Float64 sinhf64 (_Float64 __x) noexcept (true); extern _Float64 __sinhf64 (_Float64 __x) noexcept (true);
extern _Float64 tanhf64 (_Float64 __x) noexcept (true); extern _Float64 __tanhf64 (_Float64 __x) noexcept (true);
extern void sincosf64 (_Float64 __x, _Float64 *__sinx, _Float64 *__cosx) noexcept (true); extern void __sincosf64 (_Float64 __x, _Float64 *__sinx, _Float64 *__cosx) noexcept (true)
;
extern _Float64 acoshf64 (_Float64 __x) noexcept (true); extern _Float64 __acoshf64 (_Float64 __x) noexcept (true);
extern _Float64 asinhf64 (_Float64 __x) noexcept (true); extern _Float64 __asinhf64 (_Float64 __x) noexcept (true);
extern _Float64 atanhf64 (_Float64 __x) noexcept (true); extern _Float64 __atanhf64 (_Float64 __x) noexcept (true);
extern _Float64 expf64 (_Float64 __x) noexcept (true); extern _Float64 __expf64 (_Float64 __x) noexcept (true);
extern _Float64 frexpf64 (_Float64 __x, int *__exponent) noexcept (true); extern _Float64 __frexpf64 (_Float64 __x, int *__exponent) noexcept (true);
extern _Float64 ldexpf64 (_Float64 __x, int __exponent) noexcept (true); extern _Float64 __ldexpf64 (_Float64 __x, int __exponent) noexcept (true);
extern _Float64 logf64 (_Float64 __x) noexcept (true); extern _Float64 __logf64 (_Float64 __x) noexcept (true);
extern _Float64 log10f64 (_Float64 __x) noexcept (true); extern _Float64 __log10f64 (_Float64 __x) noexcept (true);
extern _Float64 modff64 (_Float64 __x, _Float64 *__iptr) noexcept (true); extern _Float64 __modff64 (_Float64 __x, _Float64 *__iptr) noexcept (true) __attribute__ ((__nonnull__ (2)));
extern _Float64 exp10f64 (_Float64 __x) noexcept (true); extern _Float64 __exp10f64 (_Float64 __x) noexcept (true);
extern _Float64 expm1f64 (_Float64 __x) noexcept (true); extern _Float64 __expm1f64 (_Float64 __x) noexcept (true);
extern _Float64 log1pf64 (_Float64 __x) noexcept (true); extern _Float64 __log1pf64 (_Float64 __x) noexcept (true);
extern _Float64 logbf64 (_Float64 __x) noexcept (true); extern _Float64 __logbf64 (_Float64 __x) noexcept (true);
extern _Float64 exp2f64 (_Float64 __x) noexcept (true); extern _Float64 __exp2f64 (_Float64 __x) noexcept (true);
extern _Float64 log2f64 (_Float64 __x) noexcept (true); extern _Float64 __log2f64 (_Float64 __x) noexcept (true);
extern _Float64 powf64 (_Float64 __x, _Float64 __y) noexcept (true); extern _Float64 __powf64 (_Float64 __x, _Float64 __y) noexcept (true);
extern _Float64 sqrtf64 (_Float64 __x) noexcept (true); extern _Float64 __sqrtf64 (_Float64 __x) noexcept (true);
extern _Float64 hypotf64 (_Float64 __x, _Float64 __y) noexcept (true); extern _Float64 __hypotf64 (_Float64 __x, _Float64 __y) noexcept (true);
extern _Float64 cbrtf64 (_Float64 __x) noexcept (true); extern _Float64 __cbrtf64 (_Float64 __x) noexcept (true);
extern _Float64 ceilf64 (_Float64 __x) noexcept (true) __attribute__ ((__const__)); extern _Float64 __ceilf64 (_Float64 __x) noexcept (true) __attribute__ ((__const__));
extern _Float64 fabsf64 (_Float64 __x) noexcept (true) __attribute__ ((__const__)); extern _Float64 __fabsf64 (_Float64 __x) noexcept (true) __attribute__ ((__const__));
extern _Float64 floorf64 (_Float64 __x) noexcept (true) __attribute__ ((__const__)); extern _Float64 __floorf64 (_Float64 __x) noexcept (true) __attribute__ ((__const__));
extern _Float64 fmodf64 (_Float64 __x, _Float64 __y) noexcept (true); extern _Float64 __fmodf64 (_Float64 __x, _Float64 __y) noexcept (true);
extern _Float64 copysignf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__)); extern _Float64 __copysignf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__));
extern _Float64 nanf64 (const char *__tagb) noexcept (true); extern _Float64 __nanf64 (const char *__tagb) noexcept (true);
extern _Float64 j0f64 (_Float64) noexcept (true); extern _Float64 __j0f64 (_Float64) noexcept (true);
extern _Float64 j1f64 (_Float64) noexcept (true); extern _Float64 __j1f64 (_Float64) noexcept (true);
extern _Float64 jnf64 (int, _Float64) noexcept (true); extern _Float64 __jnf64 (int, _Float64) noexcept (true);
extern _Float64 y0f64 (_Float64) noexcept (true); extern _Float64 __y0f64 (_Float64) noexcept (true);
extern _Float64 y1f64 (_Float64) noexcept (true); extern _Float64 __y1f64 (_Float64) noexcept (true);
extern _Float64 ynf64 (int, _Float64) noexcept (true); extern _Float64 __ynf64 (int, _Float64) noexcept (true);
extern _Float64 erff64 (_Float64) noexcept (true); extern _Float64 __erff64 (_Float64) noexcept (true);
extern _Float64 erfcf64 (_Float64) noexcept (true); extern _Float64 __erfcf64 (_Float64) noexcept (true);
extern _Float64 lgammaf64 (_Float64) noexcept (true); extern _Float64 __lgammaf64 (_Float64) noexcept (true);
extern _Float64 tgammaf64 (_Float64) noexcept (true); extern _Float64 __tgammaf64 (_Float64) noexcept (true);
extern _Float64 lgammaf64_r (_Float64, int *__signgamp) noexcept (true); extern _Float64 __lgammaf64_r (_Float64, int *__signgamp) noexcept (true);
extern _Float64 rintf64 (_Float64 __x) noexcept (true); extern _Float64 __rintf64 (_Float64 __x) noexcept (true);
extern _Float64 nextafterf64 (_Float64 __x, _Float64 __y) noexcept (true); extern _Float64 __nextafterf64 (_Float64 __x, _Float64 __y) noexcept (true);
extern _Float64 nextdownf64 (_Float64 __x) noexcept (true); extern _Float64 __nextdownf64 (_Float64 __x) noexcept (true);
extern _Float64 nextupf64 (_Float64 __x) noexcept (true); extern _Float64 __nextupf64 (_Float64 __x) noexcept (true);
extern _Float64 remainderf64 (_Float64 __x, _Float64 __y) noexcept (true); extern _Float64 __remainderf64 (_Float64 __x, _Float64 __y) noexcept (true);
extern _Float64 scalbnf64 (_Float64 __x, int __n) noexcept (true); extern _Float64 __scalbnf64 (_Float64 __x, int __n) noexcept (true);
extern int ilogbf64 (_Float64 __x) noexcept (true); extern int __ilogbf64 (_Float64 __x) noexcept (true);
extern long int llogbf64 (_Float64 __x) noexcept (true); extern long int __llogbf64 (_Float64 __x) noexcept (true);
extern _Float64 scalblnf64 (_Float64 __x, long int __n) noexcept (true); extern _Float64 __scalblnf64 (_Float64 __x, long int __n) noexcept (true);
extern _Float64 nearbyintf64 (_Float64 __x) noexcept (true); extern _Float64 __nearbyintf64 (_Float64 __x) noexcept (true);
extern _Float64 roundf64 (_Float64 __x) noexcept (true) __attribute__ ((__const__)); extern _Float64 __roundf64 (_Float64 __x) noexcept (true) __attribute__ ((__const__));
extern _Float64 truncf64 (_Float64 __x) noexcept (true) __attribute__ ((__const__)); extern _Float64 __truncf64 (_Float64 __x) noexcept (true) __attribute__ ((__const__));
extern _Float64 remquof64 (_Float64 __x, _Float64 __y, int *__quo) noexcept (true); extern _Float64 __remquof64 (_Float64 __x, _Float64 __y, int *__quo) noexcept (true);
extern long int lrintf64 (_Float64 __x) noexcept (true); extern long int __lrintf64 (_Float64 __x) noexcept (true);
__extension__
extern long long int llrintf64 (_Float64 __x) noexcept (true); extern long long int __llrintf64 (_Float64 __x) noexcept (true);
extern long int lroundf64 (_Float64 __x) noexcept (true); extern long int __lroundf64 (_Float64 __x) noexcept (true);
__extension__
extern long long int llroundf64 (_Float64 __x) noexcept (true); extern long long int __llroundf64 (_Float64 __x) noexcept (true);
extern _Float64 fdimf64 (_Float64 __x, _Float64 __y) noexcept (true); extern _Float64 __fdimf64 (_Float64 __x, _Float64 __y) noexcept (true);
extern _Float64 fmaxf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__)); extern _Float64 __fmaxf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__));
extern _Float64 fminf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__)); extern _Float64 __fminf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__));
extern _Float64 fmaf64 (_Float64 __x, _Float64 __y, _Float64 __z) noexcept (true); extern _Float64 __fmaf64 (_Float64 __x, _Float64 __y, _Float64 __z) noexcept (true);
extern _Float64 roundevenf64 (_Float64 __x) noexcept (true) __attribute__ ((__const__)); extern _Float64 __roundevenf64 (_Float64 __x) noexcept (true) __attribute__ ((__const__));
extern __intmax_t fromfpf64 (_Float64 __x, int __round, unsigned int __width) noexcept (true); extern __intmax_t __fromfpf64 (_Float64 __x, int __round, unsigned int __width) noexcept (true)
;
extern __uintmax_t ufromfpf64 (_Float64 __x, int __round, unsigned int __width) noexcept (true); extern __uintmax_t __ufromfpf64 (_Float64 __x, int __round, unsigned int __width) noexcept (true)
;
extern __intmax_t fromfpxf64 (_Float64 __x, int __round, unsigned int __width) noexcept (true); extern __intmax_t __fromfpxf64 (_Float64 __x, int __round, unsigned int __width) noexcept (true)
;
extern __uintmax_t ufromfpxf64 (_Float64 __x, int __round, unsigned int __width) noexcept (true); extern __uintmax_t __ufromfpxf64 (_Float64 __x, int __round, unsigned int __width) noexcept (true)
;
extern int canonicalizef64 (_Float64 *__cx, const _Float64 *__x) noexcept (true);
extern _Float64 fmaxmagf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__)); extern _Float64 __fmaxmagf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__));
extern _Float64 fminmagf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__)); extern _Float64 __fminmagf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__));
extern _Float64 fmaximumf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__)); extern _Float64 __fmaximumf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__));
extern _Float64 fminimumf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__)); extern _Float64 __fminimumf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__));
extern _Float64 fmaximum_numf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__)); extern _Float64 __fmaximum_numf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__));
extern _Float64 fminimum_numf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__)); extern _Float64 __fminimum_numf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__));
extern _Float64 fmaximum_magf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__)); extern _Float64 __fmaximum_magf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__));
extern _Float64 fminimum_magf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__)); extern _Float64 __fminimum_magf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__));
extern _Float64 fmaximum_mag_numf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__)); extern _Float64 __fmaximum_mag_numf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__));
extern _Float64 fminimum_mag_numf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__)); extern _Float64 __fminimum_mag_numf64 (_Float64 __x, _Float64 __y) noexcept (true) __attribute__ ((__const__));
extern int totalorderf64 (const _Float64 *__x, const _Float64 *__y) noexcept (true)
__attribute__ ((__pure__));
extern int totalordermagf64 (const _Float64 *__x, const _Float64 *__y) noexcept (true)
__attribute__ ((__pure__));
extern _Float64 getpayloadf64 (const _Float64 *__x) noexcept (true); extern _Float64 __getpayloadf64 (const _Float64 *__x) noexcept (true);
extern int setpayloadf64 (_Float64 *__x, _Float64 __payload) noexcept (true);
extern int setpayloadsigf64 (_Float64 *__x, _Float64 __payload) noexcept (true);
extern int __fpclassifyf128 (_Float128 __value) noexcept (true)
__attribute__ ((__const__));
extern int __signbitf128 (_Float128 __value) noexcept (true)
__attribute__ ((__const__));
extern int __isinff128 (_Float128 __value) noexcept (true)
__attribute__ ((__const__));
extern int __finitef128 (_Float128 __value) noexcept (true)
__attribute__ ((__const__));
extern int __isnanf128 (_Float128 __value) noexcept (true)
__attribute__ ((__const__));
extern int __iseqsigf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern int __issignalingf128 (_Float128 __value) noexcept (true)
__attribute__ ((__const__));
extern _Float128 acosf128 (_Float128 __x) noexcept (true); extern _Float128 __acosf128 (_Float128 __x) noexcept (true);
extern _Float128 asinf128 (_Float128 __x) noexcept (true); extern _Float128 __asinf128 (_Float128 __x) noexcept (true);
extern _Float128 atanf128 (_Float128 __x) noexcept (true); extern _Float128 __atanf128 (_Float128 __x) noexcept (true);
extern _Float128 atan2f128 (_Float128 __y, _Float128 __x) noexcept (true); extern _Float128 __atan2f128 (_Float128 __y, _Float128 __x) noexcept (true);
extern _Float128 cosf128 (_Float128 __x) noexcept (true); extern _Float128 __cosf128 (_Float128 __x) noexcept (true);
extern _Float128 sinf128 (_Float128 __x) noexcept (true); extern _Float128 __sinf128 (_Float128 __x) noexcept (true);
extern _Float128 tanf128 (_Float128 __x) noexcept (true); extern _Float128 __tanf128 (_Float128 __x) noexcept (true);
extern _Float128 coshf128 (_Float128 __x) noexcept (true); extern _Float128 __coshf128 (_Float128 __x) noexcept (true);
extern _Float128 sinhf128 (_Float128 __x) noexcept (true); extern _Float128 __sinhf128 (_Float128 __x) noexcept (true);
extern _Float128 tanhf128 (_Float128 __x) noexcept (true); extern _Float128 __tanhf128 (_Float128 __x) noexcept (true);
extern void sincosf128 (_Float128 __x, _Float128 *__sinx, _Float128 *__cosx) noexcept (true); extern void __sincosf128 (_Float128 __x, _Float128 *__sinx, _Float128 *__cosx) noexcept (true)
;
extern _Float128 acoshf128 (_Float128 __x) noexcept (true); extern _Float128 __acoshf128 (_Float128 __x) noexcept (true);
extern _Float128 asinhf128 (_Float128 __x) noexcept (true); extern _Float128 __asinhf128 (_Float128 __x) noexcept (true);
extern _Float128 atanhf128 (_Float128 __x) noexcept (true); extern _Float128 __atanhf128 (_Float128 __x) noexcept (true);
extern _Float128 expf128 (_Float128 __x) noexcept (true); extern _Float128 __expf128 (_Float128 __x) noexcept (true);
extern _Float128 frexpf128 (_Float128 __x, int *__exponent) noexcept (true); extern _Float128 __frexpf128 (_Float128 __x, int *__exponent) noexcept (true);
extern _Float128 ldexpf128 (_Float128 __x, int __exponent) noexcept (true); extern _Float128 __ldexpf128 (_Float128 __x, int __exponent) noexcept (true);
extern _Float128 logf128 (_Float128 __x) noexcept (true); extern _Float128 __logf128 (_Float128 __x) noexcept (true);
extern _Float128 log10f128 (_Float128 __x) noexcept (true); extern _Float128 __log10f128 (_Float128 __x) noexcept (true);
extern _Float128 modff128 (_Float128 __x, _Float128 *__iptr) noexcept (true); extern _Float128 __modff128 (_Float128 __x, _Float128 *__iptr) noexcept (true) __attribute__ ((__nonnull__ (2)));
extern _Float128 exp10f128 (_Float128 __x) noexcept (true); extern _Float128 __exp10f128 (_Float128 __x) noexcept (true);
extern _Float128 expm1f128 (_Float128 __x) noexcept (true); extern _Float128 __expm1f128 (_Float128 __x) noexcept (true);
extern _Float128 log1pf128 (_Float128 __x) noexcept (true); extern _Float128 __log1pf128 (_Float128 __x) noexcept (true);
extern _Float128 logbf128 (_Float128 __x) noexcept (true); extern _Float128 __logbf128 (_Float128 __x) noexcept (true);
extern _Float128 exp2f128 (_Float128 __x) noexcept (true); extern _Float128 __exp2f128 (_Float128 __x) noexcept (true);
extern _Float128 log2f128 (_Float128 __x) noexcept (true); extern _Float128 __log2f128 (_Float128 __x) noexcept (true);
extern _Float128 powf128 (_Float128 __x, _Float128 __y) noexcept (true); extern _Float128 __powf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float128 sqrtf128 (_Float128 __x) noexcept (true); extern _Float128 __sqrtf128 (_Float128 __x) noexcept (true);
extern _Float128 hypotf128 (_Float128 __x, _Float128 __y) noexcept (true); extern _Float128 __hypotf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float128 cbrtf128 (_Float128 __x) noexcept (true); extern _Float128 __cbrtf128 (_Float128 __x) noexcept (true);
extern _Float128 ceilf128 (_Float128 __x) noexcept (true) __attribute__ ((__const__)); extern _Float128 __ceilf128 (_Float128 __x) noexcept (true) __attribute__ ((__const__));
extern _Float128 fabsf128 (_Float128 __x) noexcept (true) __attribute__ ((__const__)); extern _Float128 __fabsf128 (_Float128 __x) noexcept (true) __attribute__ ((__const__));
extern _Float128 floorf128 (_Float128 __x) noexcept (true) __attribute__ ((__const__)); extern _Float128 __floorf128 (_Float128 __x) noexcept (true) __attribute__ ((__const__));
extern _Float128 fmodf128 (_Float128 __x, _Float128 __y) noexcept (true); extern _Float128 __fmodf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float128 copysignf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__)); extern _Float128 __copysignf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__));
extern _Float128 nanf128 (const char *__tagb) noexcept (true); extern _Float128 __nanf128 (const char *__tagb) noexcept (true);
extern _Float128 j0f128 (_Float128) noexcept (true); extern _Float128 __j0f128 (_Float128) noexcept (true);
extern _Float128 j1f128 (_Float128) noexcept (true); extern _Float128 __j1f128 (_Float128) noexcept (true);
extern _Float128 jnf128 (int, _Float128) noexcept (true); extern _Float128 __jnf128 (int, _Float128) noexcept (true);
extern _Float128 y0f128 (_Float128) noexcept (true); extern _Float128 __y0f128 (_Float128) noexcept (true);
extern _Float128 y1f128 (_Float128) noexcept (true); extern _Float128 __y1f128 (_Float128) noexcept (true);
extern _Float128 ynf128 (int, _Float128) noexcept (true); extern _Float128 __ynf128 (int, _Float128) noexcept (true);
extern _Float128 erff128 (_Float128) noexcept (true); extern _Float128 __erff128 (_Float128) noexcept (true);
extern _Float128 erfcf128 (_Float128) noexcept (true); extern _Float128 __erfcf128 (_Float128) noexcept (true);
extern _Float128 lgammaf128 (_Float128) noexcept (true); extern _Float128 __lgammaf128 (_Float128) noexcept (true);
extern _Float128 tgammaf128 (_Float128) noexcept (true); extern _Float128 __tgammaf128 (_Float128) noexcept (true);
extern _Float128 lgammaf128_r (_Float128, int *__signgamp) noexcept (true); extern _Float128 __lgammaf128_r (_Float128, int *__signgamp) noexcept (true);
extern _Float128 rintf128 (_Float128 __x) noexcept (true); extern _Float128 __rintf128 (_Float128 __x) noexcept (true);
extern _Float128 nextafterf128 (_Float128 __x, _Float128 __y) noexcept (true); extern _Float128 __nextafterf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float128 nextdownf128 (_Float128 __x) noexcept (true); extern _Float128 __nextdownf128 (_Float128 __x) noexcept (true);
extern _Float128 nextupf128 (_Float128 __x) noexcept (true); extern _Float128 __nextupf128 (_Float128 __x) noexcept (true);
extern _Float128 remainderf128 (_Float128 __x, _Float128 __y) noexcept (true); extern _Float128 __remainderf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float128 scalbnf128 (_Float128 __x, int __n) noexcept (true); extern _Float128 __scalbnf128 (_Float128 __x, int __n) noexcept (true);
extern int ilogbf128 (_Float128 __x) noexcept (true); extern int __ilogbf128 (_Float128 __x) noexcept (true);
extern long int llogbf128 (_Float128 __x) noexcept (true); extern long int __llogbf128 (_Float128 __x) noexcept (true);
extern _Float128 scalblnf128 (_Float128 __x, long int __n) noexcept (true); extern _Float128 __scalblnf128 (_Float128 __x, long int __n) noexcept (true);
extern _Float128 nearbyintf128 (_Float128 __x) noexcept (true); extern _Float128 __nearbyintf128 (_Float128 __x) noexcept (true);
extern _Float128 roundf128 (_Float128 __x) noexcept (true) __attribute__ ((__const__)); extern _Float128 __roundf128 (_Float128 __x) noexcept (true) __attribute__ ((__const__));
extern _Float128 truncf128 (_Float128 __x) noexcept (true) __attribute__ ((__const__)); extern _Float128 __truncf128 (_Float128 __x) noexcept (true) __attribute__ ((__const__));
extern _Float128 remquof128 (_Float128 __x, _Float128 __y, int *__quo) noexcept (true); extern _Float128 __remquof128 (_Float128 __x, _Float128 __y, int *__quo) noexcept (true);
extern long int lrintf128 (_Float128 __x) noexcept (true); extern long int __lrintf128 (_Float128 __x) noexcept (true);
__extension__
extern long long int llrintf128 (_Float128 __x) noexcept (true); extern long long int __llrintf128 (_Float128 __x) noexcept (true);
extern long int lroundf128 (_Float128 __x) noexcept (true); extern long int __lroundf128 (_Float128 __x) noexcept (true);
__extension__
extern long long int llroundf128 (_Float128 __x) noexcept (true); extern long long int __llroundf128 (_Float128 __x) noexcept (true);
extern _Float128 fdimf128 (_Float128 __x, _Float128 __y) noexcept (true); extern _Float128 __fdimf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float128 fmaxf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__)); extern _Float128 __fmaxf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__));
extern _Float128 fminf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__)); extern _Float128 __fminf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__));
extern _Float128 fmaf128 (_Float128 __x, _Float128 __y, _Float128 __z) noexcept (true); extern _Float128 __fmaf128 (_Float128 __x, _Float128 __y, _Float128 __z) noexcept (true);
extern _Float128 roundevenf128 (_Float128 __x) noexcept (true) __attribute__ ((__const__)); extern _Float128 __roundevenf128 (_Float128 __x) noexcept (true) __attribute__ ((__const__));
extern __intmax_t fromfpf128 (_Float128 __x, int __round, unsigned int __width) noexcept (true); extern __intmax_t __fromfpf128 (_Float128 __x, int __round, unsigned int __width) noexcept (true)
;
extern __uintmax_t ufromfpf128 (_Float128 __x, int __round, unsigned int __width) noexcept (true); extern __uintmax_t __ufromfpf128 (_Float128 __x, int __round, unsigned int __width) noexcept (true)
;
extern __intmax_t fromfpxf128 (_Float128 __x, int __round, unsigned int __width) noexcept (true); extern __intmax_t __fromfpxf128 (_Float128 __x, int __round, unsigned int __width) noexcept (true)
;
extern __uintmax_t ufromfpxf128 (_Float128 __x, int __round, unsigned int __width) noexcept (true); extern __uintmax_t __ufromfpxf128 (_Float128 __x, int __round, unsigned int __width) noexcept (true)
;
extern int canonicalizef128 (_Float128 *__cx, const _Float128 *__x) noexcept (true);
extern _Float128 fmaxmagf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__)); extern _Float128 __fmaxmagf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__));
extern _Float128 fminmagf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__)); extern _Float128 __fminmagf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__));
extern _Float128 fmaximumf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__)); extern _Float128 __fmaximumf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__));
extern _Float128 fminimumf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__)); extern _Float128 __fminimumf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__));
extern _Float128 fmaximum_numf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__)); extern _Float128 __fmaximum_numf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__));
extern _Float128 fminimum_numf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__)); extern _Float128 __fminimum_numf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__));
extern _Float128 fmaximum_magf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__)); extern _Float128 __fmaximum_magf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__));
extern _Float128 fminimum_magf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__)); extern _Float128 __fminimum_magf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__));
extern _Float128 fmaximum_mag_numf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__)); extern _Float128 __fmaximum_mag_numf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__));
extern _Float128 fminimum_mag_numf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__)); extern _Float128 __fminimum_mag_numf128 (_Float128 __x, _Float128 __y) noexcept (true) __attribute__ ((__const__));
extern int totalorderf128 (const _Float128 *__x, const _Float128 *__y) noexcept (true)
__attribute__ ((__pure__));
extern int totalordermagf128 (const _Float128 *__x, const _Float128 *__y) noexcept (true)
__attribute__ ((__pure__));
extern _Float128 getpayloadf128 (const _Float128 *__x) noexcept (true); extern _Float128 __getpayloadf128 (const _Float128 *__x) noexcept (true);
extern int setpayloadf128 (_Float128 *__x, _Float128 __payload) noexcept (true);
extern int setpayloadsigf128 (_Float128 *__x, _Float128 __payload) noexcept (true);
extern _Float32x acosf32x (_Float32x __x) noexcept (true); extern _Float32x __acosf32x (_Float32x __x) noexcept (true);
extern _Float32x asinf32x (_Float32x __x) noexcept (true); extern _Float32x __asinf32x (_Float32x __x) noexcept (true);
extern _Float32x atanf32x (_Float32x __x) noexcept (true); extern _Float32x __atanf32x (_Float32x __x) noexcept (true);
extern _Float32x atan2f32x (_Float32x __y, _Float32x __x) noexcept (true); extern _Float32x __atan2f32x (_Float32x __y, _Float32x __x) noexcept (true);
extern _Float32x cosf32x (_Float32x __x) noexcept (true); extern _Float32x __cosf32x (_Float32x __x) noexcept (true);
extern _Float32x sinf32x (_Float32x __x) noexcept (true); extern _Float32x __sinf32x (_Float32x __x) noexcept (true);
extern _Float32x tanf32x (_Float32x __x) noexcept (true); extern _Float32x __tanf32x (_Float32x __x) noexcept (true);
extern _Float32x coshf32x (_Float32x __x) noexcept (true); extern _Float32x __coshf32x (_Float32x __x) noexcept (true);
extern _Float32x sinhf32x (_Float32x __x) noexcept (true); extern _Float32x __sinhf32x (_Float32x __x) noexcept (true);
extern _Float32x tanhf32x (_Float32x __x) noexcept (true); extern _Float32x __tanhf32x (_Float32x __x) noexcept (true);
extern void sincosf32x (_Float32x __x, _Float32x *__sinx, _Float32x *__cosx) noexcept (true); extern void __sincosf32x (_Float32x __x, _Float32x *__sinx, _Float32x *__cosx) noexcept (true)
;
extern _Float32x acoshf32x (_Float32x __x) noexcept (true); extern _Float32x __acoshf32x (_Float32x __x) noexcept (true);
extern _Float32x asinhf32x (_Float32x __x) noexcept (true); extern _Float32x __asinhf32x (_Float32x __x) noexcept (true);
extern _Float32x atanhf32x (_Float32x __x) noexcept (true); extern _Float32x __atanhf32x (_Float32x __x) noexcept (true);
extern _Float32x expf32x (_Float32x __x) noexcept (true); extern _Float32x __expf32x (_Float32x __x) noexcept (true);
extern _Float32x frexpf32x (_Float32x __x, int *__exponent) noexcept (true); extern _Float32x __frexpf32x (_Float32x __x, int *__exponent) noexcept (true);
extern _Float32x ldexpf32x (_Float32x __x, int __exponent) noexcept (true); extern _Float32x __ldexpf32x (_Float32x __x, int __exponent) noexcept (true);
extern _Float32x logf32x (_Float32x __x) noexcept (true); extern _Float32x __logf32x (_Float32x __x) noexcept (true);
extern _Float32x log10f32x (_Float32x __x) noexcept (true); extern _Float32x __log10f32x (_Float32x __x) noexcept (true);
extern _Float32x modff32x (_Float32x __x, _Float32x *__iptr) noexcept (true); extern _Float32x __modff32x (_Float32x __x, _Float32x *__iptr) noexcept (true) __attribute__ ((__nonnull__ (2)));
extern _Float32x exp10f32x (_Float32x __x) noexcept (true); extern _Float32x __exp10f32x (_Float32x __x) noexcept (true);
extern _Float32x expm1f32x (_Float32x __x) noexcept (true); extern _Float32x __expm1f32x (_Float32x __x) noexcept (true);
extern _Float32x log1pf32x (_Float32x __x) noexcept (true); extern _Float32x __log1pf32x (_Float32x __x) noexcept (true);
extern _Float32x logbf32x (_Float32x __x) noexcept (true); extern _Float32x __logbf32x (_Float32x __x) noexcept (true);
extern _Float32x exp2f32x (_Float32x __x) noexcept (true); extern _Float32x __exp2f32x (_Float32x __x) noexcept (true);
extern _Float32x log2f32x (_Float32x __x) noexcept (true); extern _Float32x __log2f32x (_Float32x __x) noexcept (true);
extern _Float32x powf32x (_Float32x __x, _Float32x __y) noexcept (true); extern _Float32x __powf32x (_Float32x __x, _Float32x __y) noexcept (true);
extern _Float32x sqrtf32x (_Float32x __x) noexcept (true); extern _Float32x __sqrtf32x (_Float32x __x) noexcept (true);
extern _Float32x hypotf32x (_Float32x __x, _Float32x __y) noexcept (true); extern _Float32x __hypotf32x (_Float32x __x, _Float32x __y) noexcept (true);
extern _Float32x cbrtf32x (_Float32x __x) noexcept (true); extern _Float32x __cbrtf32x (_Float32x __x) noexcept (true);
extern _Float32x ceilf32x (_Float32x __x) noexcept (true) __attribute__ ((__const__)); extern _Float32x __ceilf32x (_Float32x __x) noexcept (true) __attribute__ ((__const__));
extern _Float32x fabsf32x (_Float32x __x) noexcept (true) __attribute__ ((__const__)); extern _Float32x __fabsf32x (_Float32x __x) noexcept (true) __attribute__ ((__const__));
extern _Float32x floorf32x (_Float32x __x) noexcept (true) __attribute__ ((__const__)); extern _Float32x __floorf32x (_Float32x __x) noexcept (true) __attribute__ ((__const__));
extern _Float32x fmodf32x (_Float32x __x, _Float32x __y) noexcept (true); extern _Float32x __fmodf32x (_Float32x __x, _Float32x __y) noexcept (true);
extern _Float32x copysignf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__)); extern _Float32x __copysignf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__));
extern _Float32x nanf32x (const char *__tagb) noexcept (true); extern _Float32x __nanf32x (const char *__tagb) noexcept (true);
extern _Float32x j0f32x (_Float32x) noexcept (true); extern _Float32x __j0f32x (_Float32x) noexcept (true);
extern _Float32x j1f32x (_Float32x) noexcept (true); extern _Float32x __j1f32x (_Float32x) noexcept (true);
extern _Float32x jnf32x (int, _Float32x) noexcept (true); extern _Float32x __jnf32x (int, _Float32x) noexcept (true);
extern _Float32x y0f32x (_Float32x) noexcept (true); extern _Float32x __y0f32x (_Float32x) noexcept (true);
extern _Float32x y1f32x (_Float32x) noexcept (true); extern _Float32x __y1f32x (_Float32x) noexcept (true);
extern _Float32x ynf32x (int, _Float32x) noexcept (true); extern _Float32x __ynf32x (int, _Float32x) noexcept (true);
extern _Float32x erff32x (_Float32x) noexcept (true); extern _Float32x __erff32x (_Float32x) noexcept (true);
extern _Float32x erfcf32x (_Float32x) noexcept (true); extern _Float32x __erfcf32x (_Float32x) noexcept (true);
extern _Float32x lgammaf32x (_Float32x) noexcept (true); extern _Float32x __lgammaf32x (_Float32x) noexcept (true);
extern _Float32x tgammaf32x (_Float32x) noexcept (true); extern _Float32x __tgammaf32x (_Float32x) noexcept (true);
extern _Float32x lgammaf32x_r (_Float32x, int *__signgamp) noexcept (true); extern _Float32x __lgammaf32x_r (_Float32x, int *__signgamp) noexcept (true);
extern _Float32x rintf32x (_Float32x __x) noexcept (true); extern _Float32x __rintf32x (_Float32x __x) noexcept (true);
extern _Float32x nextafterf32x (_Float32x __x, _Float32x __y) noexcept (true); extern _Float32x __nextafterf32x (_Float32x __x, _Float32x __y) noexcept (true);
extern _Float32x nextdownf32x (_Float32x __x) noexcept (true); extern _Float32x __nextdownf32x (_Float32x __x) noexcept (true);
extern _Float32x nextupf32x (_Float32x __x) noexcept (true); extern _Float32x __nextupf32x (_Float32x __x) noexcept (true);
extern _Float32x remainderf32x (_Float32x __x, _Float32x __y) noexcept (true); extern _Float32x __remainderf32x (_Float32x __x, _Float32x __y) noexcept (true);
extern _Float32x scalbnf32x (_Float32x __x, int __n) noexcept (true); extern _Float32x __scalbnf32x (_Float32x __x, int __n) noexcept (true);
extern int ilogbf32x (_Float32x __x) noexcept (true); extern int __ilogbf32x (_Float32x __x) noexcept (true);
extern long int llogbf32x (_Float32x __x) noexcept (true); extern long int __llogbf32x (_Float32x __x) noexcept (true);
extern _Float32x scalblnf32x (_Float32x __x, long int __n) noexcept (true); extern _Float32x __scalblnf32x (_Float32x __x, long int __n) noexcept (true);
extern _Float32x nearbyintf32x (_Float32x __x) noexcept (true); extern _Float32x __nearbyintf32x (_Float32x __x) noexcept (true);
extern _Float32x roundf32x (_Float32x __x) noexcept (true) __attribute__ ((__const__)); extern _Float32x __roundf32x (_Float32x __x) noexcept (true) __attribute__ ((__const__));
extern _Float32x truncf32x (_Float32x __x) noexcept (true) __attribute__ ((__const__)); extern _Float32x __truncf32x (_Float32x __x) noexcept (true) __attribute__ ((__const__));
extern _Float32x remquof32x (_Float32x __x, _Float32x __y, int *__quo) noexcept (true); extern _Float32x __remquof32x (_Float32x __x, _Float32x __y, int *__quo) noexcept (true);
extern long int lrintf32x (_Float32x __x) noexcept (true); extern long int __lrintf32x (_Float32x __x) noexcept (true);
__extension__
extern long long int llrintf32x (_Float32x __x) noexcept (true); extern long long int __llrintf32x (_Float32x __x) noexcept (true);
extern long int lroundf32x (_Float32x __x) noexcept (true); extern long int __lroundf32x (_Float32x __x) noexcept (true);
__extension__
extern long long int llroundf32x (_Float32x __x) noexcept (true); extern long long int __llroundf32x (_Float32x __x) noexcept (true);
extern _Float32x fdimf32x (_Float32x __x, _Float32x __y) noexcept (true); extern _Float32x __fdimf32x (_Float32x __x, _Float32x __y) noexcept (true);
extern _Float32x fmaxf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__)); extern _Float32x __fmaxf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__));
extern _Float32x fminf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__)); extern _Float32x __fminf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__));
extern _Float32x fmaf32x (_Float32x __x, _Float32x __y, _Float32x __z) noexcept (true); extern _Float32x __fmaf32x (_Float32x __x, _Float32x __y, _Float32x __z) noexcept (true);
extern _Float32x roundevenf32x (_Float32x __x) noexcept (true) __attribute__ ((__const__)); extern _Float32x __roundevenf32x (_Float32x __x) noexcept (true) __attribute__ ((__const__));
extern __intmax_t fromfpf32x (_Float32x __x, int __round, unsigned int __width) noexcept (true); extern __intmax_t __fromfpf32x (_Float32x __x, int __round, unsigned int __width) noexcept (true)
;
extern __uintmax_t ufromfpf32x (_Float32x __x, int __round, unsigned int __width) noexcept (true); extern __uintmax_t __ufromfpf32x (_Float32x __x, int __round, unsigned int __width) noexcept (true)
;
extern __intmax_t fromfpxf32x (_Float32x __x, int __round, unsigned int __width) noexcept (true); extern __intmax_t __fromfpxf32x (_Float32x __x, int __round, unsigned int __width) noexcept (true)
;
extern __uintmax_t ufromfpxf32x (_Float32x __x, int __round, unsigned int __width) noexcept (true); extern __uintmax_t __ufromfpxf32x (_Float32x __x, int __round, unsigned int __width) noexcept (true)
;
extern int canonicalizef32x (_Float32x *__cx, const _Float32x *__x) noexcept (true);
extern _Float32x fmaxmagf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__)); extern _Float32x __fmaxmagf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__));
extern _Float32x fminmagf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__)); extern _Float32x __fminmagf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__));
extern _Float32x fmaximumf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__)); extern _Float32x __fmaximumf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__));
extern _Float32x fminimumf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__)); extern _Float32x __fminimumf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__));
extern _Float32x fmaximum_numf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__)); extern _Float32x __fmaximum_numf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__));
extern _Float32x fminimum_numf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__)); extern _Float32x __fminimum_numf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__));
extern _Float32x fmaximum_magf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__)); extern _Float32x __fmaximum_magf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__));
extern _Float32x fminimum_magf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__)); extern _Float32x __fminimum_magf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__));
extern _Float32x fmaximum_mag_numf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__)); extern _Float32x __fmaximum_mag_numf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__));
extern _Float32x fminimum_mag_numf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__)); extern _Float32x __fminimum_mag_numf32x (_Float32x __x, _Float32x __y) noexcept (true) __attribute__ ((__const__));
extern int totalorderf32x (const _Float32x *__x, const _Float32x *__y) noexcept (true)
__attribute__ ((__pure__));
extern int totalordermagf32x (const _Float32x *__x, const _Float32x *__y) noexcept (true)
__attribute__ ((__pure__));
extern _Float32x getpayloadf32x (const _Float32x *__x) noexcept (true); extern _Float32x __getpayloadf32x (const _Float32x *__x) noexcept (true);
extern int setpayloadf32x (_Float32x *__x, _Float32x __payload) noexcept (true);
extern int setpayloadsigf32x (_Float32x *__x, _Float32x __payload) noexcept (true);
extern _Float64x acosf64x (_Float64x __x) noexcept (true); extern _Float64x __acosf64x (_Float64x __x) noexcept (true);
extern _Float64x asinf64x (_Float64x __x) noexcept (true); extern _Float64x __asinf64x (_Float64x __x) noexcept (true);
extern _Float64x atanf64x (_Float64x __x) noexcept (true); extern _Float64x __atanf64x (_Float64x __x) noexcept (true);
extern _Float64x atan2f64x (_Float64x __y, _Float64x __x) noexcept (true); extern _Float64x __atan2f64x (_Float64x __y, _Float64x __x) noexcept (true);
extern _Float64x cosf64x (_Float64x __x) noexcept (true); extern _Float64x __cosf64x (_Float64x __x) noexcept (true);
extern _Float64x sinf64x (_Float64x __x) noexcept (true); extern _Float64x __sinf64x (_Float64x __x) noexcept (true);
extern _Float64x tanf64x (_Float64x __x) noexcept (true); extern _Float64x __tanf64x (_Float64x __x) noexcept (true);
extern _Float64x coshf64x (_Float64x __x) noexcept (true); extern _Float64x __coshf64x (_Float64x __x) noexcept (true);
extern _Float64x sinhf64x (_Float64x __x) noexcept (true); extern _Float64x __sinhf64x (_Float64x __x) noexcept (true);
extern _Float64x tanhf64x (_Float64x __x) noexcept (true); extern _Float64x __tanhf64x (_Float64x __x) noexcept (true);
extern void sincosf64x (_Float64x __x, _Float64x *__sinx, _Float64x *__cosx) noexcept (true); extern void __sincosf64x (_Float64x __x, _Float64x *__sinx, _Float64x *__cosx) noexcept (true)
;
extern _Float64x acoshf64x (_Float64x __x) noexcept (true); extern _Float64x __acoshf64x (_Float64x __x) noexcept (true);
extern _Float64x asinhf64x (_Float64x __x) noexcept (true); extern _Float64x __asinhf64x (_Float64x __x) noexcept (true);
extern _Float64x atanhf64x (_Float64x __x) noexcept (true); extern _Float64x __atanhf64x (_Float64x __x) noexcept (true);
extern _Float64x expf64x (_Float64x __x) noexcept (true); extern _Float64x __expf64x (_Float64x __x) noexcept (true);
extern _Float64x frexpf64x (_Float64x __x, int *__exponent) noexcept (true); extern _Float64x __frexpf64x (_Float64x __x, int *__exponent) noexcept (true);
extern _Float64x ldexpf64x (_Float64x __x, int __exponent) noexcept (true); extern _Float64x __ldexpf64x (_Float64x __x, int __exponent) noexcept (true);
extern _Float64x logf64x (_Float64x __x) noexcept (true); extern _Float64x __logf64x (_Float64x __x) noexcept (true);
extern _Float64x log10f64x (_Float64x __x) noexcept (true); extern _Float64x __log10f64x (_Float64x __x) noexcept (true);
extern _Float64x modff64x (_Float64x __x, _Float64x *__iptr) noexcept (true); extern _Float64x __modff64x (_Float64x __x, _Float64x *__iptr) noexcept (true) __attribute__ ((__nonnull__ (2)));
extern _Float64x exp10f64x (_Float64x __x) noexcept (true); extern _Float64x __exp10f64x (_Float64x __x) noexcept (true);
extern _Float64x expm1f64x (_Float64x __x) noexcept (true); extern _Float64x __expm1f64x (_Float64x __x) noexcept (true);
extern _Float64x log1pf64x (_Float64x __x) noexcept (true); extern _Float64x __log1pf64x (_Float64x __x) noexcept (true);
extern _Float64x logbf64x (_Float64x __x) noexcept (true); extern _Float64x __logbf64x (_Float64x __x) noexcept (true);
extern _Float64x exp2f64x (_Float64x __x) noexcept (true); extern _Float64x __exp2f64x (_Float64x __x) noexcept (true);
extern _Float64x log2f64x (_Float64x __x) noexcept (true); extern _Float64x __log2f64x (_Float64x __x) noexcept (true);
extern _Float64x powf64x (_Float64x __x, _Float64x __y) noexcept (true); extern _Float64x __powf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float64x sqrtf64x (_Float64x __x) noexcept (true); extern _Float64x __sqrtf64x (_Float64x __x) noexcept (true);
extern _Float64x hypotf64x (_Float64x __x, _Float64x __y) noexcept (true); extern _Float64x __hypotf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float64x cbrtf64x (_Float64x __x) noexcept (true); extern _Float64x __cbrtf64x (_Float64x __x) noexcept (true);
extern _Float64x ceilf64x (_Float64x __x) noexcept (true) __attribute__ ((__const__)); extern _Float64x __ceilf64x (_Float64x __x) noexcept (true) __attribute__ ((__const__));
extern _Float64x fabsf64x (_Float64x __x) noexcept (true) __attribute__ ((__const__)); extern _Float64x __fabsf64x (_Float64x __x) noexcept (true) __attribute__ ((__const__));
extern _Float64x floorf64x (_Float64x __x) noexcept (true) __attribute__ ((__const__)); extern _Float64x __floorf64x (_Float64x __x) noexcept (true) __attribute__ ((__const__));
extern _Float64x fmodf64x (_Float64x __x, _Float64x __y) noexcept (true); extern _Float64x __fmodf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float64x copysignf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__)); extern _Float64x __copysignf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__));
extern _Float64x nanf64x (const char *__tagb) noexcept (true); extern _Float64x __nanf64x (const char *__tagb) noexcept (true);
extern _Float64x j0f64x (_Float64x) noexcept (true); extern _Float64x __j0f64x (_Float64x) noexcept (true);
extern _Float64x j1f64x (_Float64x) noexcept (true); extern _Float64x __j1f64x (_Float64x) noexcept (true);
extern _Float64x jnf64x (int, _Float64x) noexcept (true); extern _Float64x __jnf64x (int, _Float64x) noexcept (true);
extern _Float64x y0f64x (_Float64x) noexcept (true); extern _Float64x __y0f64x (_Float64x) noexcept (true);
extern _Float64x y1f64x (_Float64x) noexcept (true); extern _Float64x __y1f64x (_Float64x) noexcept (true);
extern _Float64x ynf64x (int, _Float64x) noexcept (true); extern _Float64x __ynf64x (int, _Float64x) noexcept (true);
extern _Float64x erff64x (_Float64x) noexcept (true); extern _Float64x __erff64x (_Float64x) noexcept (true);
extern _Float64x erfcf64x (_Float64x) noexcept (true); extern _Float64x __erfcf64x (_Float64x) noexcept (true);
extern _Float64x lgammaf64x (_Float64x) noexcept (true); extern _Float64x __lgammaf64x (_Float64x) noexcept (true);
extern _Float64x tgammaf64x (_Float64x) noexcept (true); extern _Float64x __tgammaf64x (_Float64x) noexcept (true);
extern _Float64x lgammaf64x_r (_Float64x, int *__signgamp) noexcept (true); extern _Float64x __lgammaf64x_r (_Float64x, int *__signgamp) noexcept (true);
extern _Float64x rintf64x (_Float64x __x) noexcept (true); extern _Float64x __rintf64x (_Float64x __x) noexcept (true);
extern _Float64x nextafterf64x (_Float64x __x, _Float64x __y) noexcept (true); extern _Float64x __nextafterf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float64x nextdownf64x (_Float64x __x) noexcept (true); extern _Float64x __nextdownf64x (_Float64x __x) noexcept (true);
extern _Float64x nextupf64x (_Float64x __x) noexcept (true); extern _Float64x __nextupf64x (_Float64x __x) noexcept (true);
extern _Float64x remainderf64x (_Float64x __x, _Float64x __y) noexcept (true); extern _Float64x __remainderf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float64x scalbnf64x (_Float64x __x, int __n) noexcept (true); extern _Float64x __scalbnf64x (_Float64x __x, int __n) noexcept (true);
extern int ilogbf64x (_Float64x __x) noexcept (true); extern int __ilogbf64x (_Float64x __x) noexcept (true);
extern long int llogbf64x (_Float64x __x) noexcept (true); extern long int __llogbf64x (_Float64x __x) noexcept (true);
extern _Float64x scalblnf64x (_Float64x __x, long int __n) noexcept (true); extern _Float64x __scalblnf64x (_Float64x __x, long int __n) noexcept (true);
extern _Float64x nearbyintf64x (_Float64x __x) noexcept (true); extern _Float64x __nearbyintf64x (_Float64x __x) noexcept (true);
extern _Float64x roundf64x (_Float64x __x) noexcept (true) __attribute__ ((__const__)); extern _Float64x __roundf64x (_Float64x __x) noexcept (true) __attribute__ ((__const__));
extern _Float64x truncf64x (_Float64x __x) noexcept (true) __attribute__ ((__const__)); extern _Float64x __truncf64x (_Float64x __x) noexcept (true) __attribute__ ((__const__));
extern _Float64x remquof64x (_Float64x __x, _Float64x __y, int *__quo) noexcept (true); extern _Float64x __remquof64x (_Float64x __x, _Float64x __y, int *__quo) noexcept (true);
extern long int lrintf64x (_Float64x __x) noexcept (true); extern long int __lrintf64x (_Float64x __x) noexcept (true);
__extension__
extern long long int llrintf64x (_Float64x __x) noexcept (true); extern long long int __llrintf64x (_Float64x __x) noexcept (true);
extern long int lroundf64x (_Float64x __x) noexcept (true); extern long int __lroundf64x (_Float64x __x) noexcept (true);
__extension__
extern long long int llroundf64x (_Float64x __x) noexcept (true); extern long long int __llroundf64x (_Float64x __x) noexcept (true);
extern _Float64x fdimf64x (_Float64x __x, _Float64x __y) noexcept (true); extern _Float64x __fdimf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float64x fmaxf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__)); extern _Float64x __fmaxf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__));
extern _Float64x fminf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__)); extern _Float64x __fminf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__));
extern _Float64x fmaf64x (_Float64x __x, _Float64x __y, _Float64x __z) noexcept (true); extern _Float64x __fmaf64x (_Float64x __x, _Float64x __y, _Float64x __z) noexcept (true);
extern _Float64x roundevenf64x (_Float64x __x) noexcept (true) __attribute__ ((__const__)); extern _Float64x __roundevenf64x (_Float64x __x) noexcept (true) __attribute__ ((__const__));
extern __intmax_t fromfpf64x (_Float64x __x, int __round, unsigned int __width) noexcept (true); extern __intmax_t __fromfpf64x (_Float64x __x, int __round, unsigned int __width) noexcept (true)
;
extern __uintmax_t ufromfpf64x (_Float64x __x, int __round, unsigned int __width) noexcept (true); extern __uintmax_t __ufromfpf64x (_Float64x __x, int __round, unsigned int __width) noexcept (true)
;
extern __intmax_t fromfpxf64x (_Float64x __x, int __round, unsigned int __width) noexcept (true); extern __intmax_t __fromfpxf64x (_Float64x __x, int __round, unsigned int __width) noexcept (true)
;
extern __uintmax_t ufromfpxf64x (_Float64x __x, int __round, unsigned int __width) noexcept (true); extern __uintmax_t __ufromfpxf64x (_Float64x __x, int __round, unsigned int __width) noexcept (true)
;
extern int canonicalizef64x (_Float64x *__cx, const _Float64x *__x) noexcept (true);
extern _Float64x fmaxmagf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__)); extern _Float64x __fmaxmagf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__));
extern _Float64x fminmagf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__)); extern _Float64x __fminmagf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__));
extern _Float64x fmaximumf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__)); extern _Float64x __fmaximumf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__));
extern _Float64x fminimumf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__)); extern _Float64x __fminimumf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__));
extern _Float64x fmaximum_numf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__)); extern _Float64x __fmaximum_numf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__));
extern _Float64x fminimum_numf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__)); extern _Float64x __fminimum_numf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__));
extern _Float64x fmaximum_magf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__)); extern _Float64x __fmaximum_magf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__));
extern _Float64x fminimum_magf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__)); extern _Float64x __fminimum_magf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__));
extern _Float64x fmaximum_mag_numf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__)); extern _Float64x __fmaximum_mag_numf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__));
extern _Float64x fminimum_mag_numf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__)); extern _Float64x __fminimum_mag_numf64x (_Float64x __x, _Float64x __y) noexcept (true) __attribute__ ((__const__));
extern int totalorderf64x (const _Float64x *__x, const _Float64x *__y) noexcept (true)
__attribute__ ((__pure__));
extern int totalordermagf64x (const _Float64x *__x, const _Float64x *__y) noexcept (true)
__attribute__ ((__pure__));
extern _Float64x getpayloadf64x (const _Float64x *__x) noexcept (true); extern _Float64x __getpayloadf64x (const _Float64x *__x) noexcept (true);
extern int setpayloadf64x (_Float64x *__x, _Float64x __payload) noexcept (true);
extern int setpayloadsigf64x (_Float64x *__x, _Float64x __payload) noexcept (true);
extern float fadd (double __x, double __y) noexcept (true);
extern float fdiv (double __x, double __y) noexcept (true);
extern float ffma (double __x, double __y, double __z) noexcept (true);
extern float fmul (double __x, double __y) noexcept (true);
extern float fsqrt (double __x) noexcept (true);
extern float fsub (double __x, double __y) noexcept (true);
extern float faddl (long double __x, long double __y) noexcept (true);
extern float fdivl (long double __x, long double __y) noexcept (true);
extern float ffmal (long double __x, long double __y, long double __z) noexcept (true);
extern float fmull (long double __x, long double __y) noexcept (true);
extern float fsqrtl (long double __x) noexcept (true);
extern float fsubl (long double __x, long double __y) noexcept (true);
extern double daddl (long double __x, long double __y) noexcept (true);
extern double ddivl (long double __x, long double __y) noexcept (true);
extern double dfmal (long double __x, long double __y, long double __z) noexcept (true);
extern double dmull (long double __x, long double __y) noexcept (true);
extern double dsqrtl (long double __x) noexcept (true);
extern double dsubl (long double __x, long double __y) noexcept (true);
extern _Float32 f32addf32x (_Float32x __x, _Float32x __y) noexcept (true);
extern _Float32 f32divf32x (_Float32x __x, _Float32x __y) noexcept (true);
extern _Float32 f32fmaf32x (_Float32x __x, _Float32x __y, _Float32x __z) noexcept (true);
extern _Float32 f32mulf32x (_Float32x __x, _Float32x __y) noexcept (true);
extern _Float32 f32sqrtf32x (_Float32x __x) noexcept (true);
extern _Float32 f32subf32x (_Float32x __x, _Float32x __y) noexcept (true);
extern _Float32 f32addf64 (_Float64 __x, _Float64 __y) noexcept (true);
extern _Float32 f32divf64 (_Float64 __x, _Float64 __y) noexcept (true);
extern _Float32 f32fmaf64 (_Float64 __x, _Float64 __y, _Float64 __z) noexcept (true);
extern _Float32 f32mulf64 (_Float64 __x, _Float64 __y) noexcept (true);
extern _Float32 f32sqrtf64 (_Float64 __x) noexcept (true);
extern _Float32 f32subf64 (_Float64 __x, _Float64 __y) noexcept (true);
extern _Float32 f32addf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float32 f32divf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float32 f32fmaf64x (_Float64x __x, _Float64x __y, _Float64x __z) noexcept (true);
extern _Float32 f32mulf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float32 f32sqrtf64x (_Float64x __x) noexcept (true);
extern _Float32 f32subf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float32 f32addf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float32 f32divf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float32 f32fmaf128 (_Float128 __x, _Float128 __y, _Float128 __z) noexcept (true);
extern _Float32 f32mulf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float32 f32sqrtf128 (_Float128 __x) noexcept (true);
extern _Float32 f32subf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float32x f32xaddf64 (_Float64 __x, _Float64 __y) noexcept (true);
extern _Float32x f32xdivf64 (_Float64 __x, _Float64 __y) noexcept (true);
extern _Float32x f32xfmaf64 (_Float64 __x, _Float64 __y, _Float64 __z) noexcept (true);
extern _Float32x f32xmulf64 (_Float64 __x, _Float64 __y) noexcept (true);
extern _Float32x f32xsqrtf64 (_Float64 __x) noexcept (true);
extern _Float32x f32xsubf64 (_Float64 __x, _Float64 __y) noexcept (true);
extern _Float32x f32xaddf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float32x f32xdivf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float32x f32xfmaf64x (_Float64x __x, _Float64x __y, _Float64x __z) noexcept (true);
extern _Float32x f32xmulf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float32x f32xsqrtf64x (_Float64x __x) noexcept (true);
extern _Float32x f32xsubf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float32x f32xaddf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float32x f32xdivf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float32x f32xfmaf128 (_Float128 __x, _Float128 __y, _Float128 __z) noexcept (true);
extern _Float32x f32xmulf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float32x f32xsqrtf128 (_Float128 __x) noexcept (true);
extern _Float32x f32xsubf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float64 f64addf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float64 f64divf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float64 f64fmaf64x (_Float64x __x, _Float64x __y, _Float64x __z) noexcept (true);
extern _Float64 f64mulf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float64 f64sqrtf64x (_Float64x __x) noexcept (true);
extern _Float64 f64subf64x (_Float64x __x, _Float64x __y) noexcept (true);
extern _Float64 f64addf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float64 f64divf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float64 f64fmaf128 (_Float128 __x, _Float128 __y, _Float128 __z) noexcept (true);
extern _Float64 f64mulf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float64 f64sqrtf128 (_Float128 __x) noexcept (true);
extern _Float64 f64subf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float64x f64xaddf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float64x f64xdivf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float64x f64xfmaf128 (_Float128 __x, _Float128 __y, _Float128 __z) noexcept (true);
extern _Float64x f64xmulf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern _Float64x f64xsqrtf128 (_Float128 __x) noexcept (true);
extern _Float64x f64xsubf128 (_Float128 __x, _Float128 __y) noexcept (true);
extern int signgam;
enum
{
FP_NAN =
0,
FP_INFINITE =
1,
FP_ZERO =
2,
FP_SUBNORMAL =
3,
FP_NORMAL =
4
};
extern int __iscanonicall (long double __x)
noexcept (true) __attribute__ ((__const__));
extern "C++" {
inline int iscanonical (float __val) { return ((void) (__typeof (__val)) (__val), 1); }
inline int iscanonical (double __val) { return ((void) (__typeof (__val)) (__val), 1); }
inline int iscanonical (long double __val) { return __iscanonicall (__val); }
inline int iscanonical (_Float128 __val) { return ((void) (__typeof (__val)) (__val), 1); }
}
extern "C++" {
inline int issignaling (float __val) { return __issignalingf (__val); }
inline int issignaling (double __val) { return __issignaling (__val); }
inline int
issignaling (long double __val)
{
return __issignalingl (__val);
}
inline int issignaling (_Float128 __val) { return __issignalingf128 (__val); }
}
extern "C++" {
template <class __T> inline bool
iszero (__T __val)
{
return __val == 0;
}
}
extern "C++" {
template<typename> struct __iseqsig_type;
template<> struct __iseqsig_type<float>
{
static int __call (float __x, float __y) throw ()
{
return __iseqsigf (__x, __y);
}
};
template<> struct __iseqsig_type<double>
{
static int __call (double __x, double __y) throw ()
{
return __iseqsig (__x, __y);
}
};
template<> struct __iseqsig_type<long double>
{
static int __call (long double __x, long double __y) throw ()
{
return __iseqsigl (__x, __y);
}
};
template<> struct __iseqsig_type<_Float32>
{
static int __call (_Float32 __x, _Float32 __y) throw ()
{
return __iseqsigf (__x, __y);
}
};
template<> struct __iseqsig_type<_Float64>
{
static int __call (_Float64 __x, _Float64 __y) throw ()
{
return __iseqsig (__x, __y);
}
};
template<> struct __iseqsig_type<_Float128>
{
static int __call (_Float128 __x, _Float128 __y) throw ()
{
return __iseqsigf128 (__x, __y);
}
};
template<> struct __iseqsig_type<_Float32x>
{
static int __call (_Float32x __x, _Float32x __y) throw ()
{
return __iseqsig (__x, __y);
}
};
template<> struct __iseqsig_type<_Float64x>
{
static int __call (_Float64x __x, _Float64x __y) throw ()
{
return __iseqsigl (__x, __y);
}
};
template<typename _T1, typename _T2>
inline int
iseqsig (_T1 __x, _T2 __y) throw ()
{
typedef decltype (((__x) + (__y) + 0.0L)) _T3;
return __iseqsig_type<_T3>::__call (__x, __y);
}
}
}
extern "C" {
typedef struct
{
int quot;
int rem;
} div_t;
typedef struct
{
long int quot;
long int rem;
} ldiv_t;
__extension__ typedef struct
{
long long int quot;
long long int rem;
} lldiv_t;
extern size_t __ctype_get_mb_cur_max (void) noexcept (true) ;
extern double atof (const char *__nptr)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1))) ;
extern int atoi (const char *__nptr)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1))) ;
extern long int atol (const char *__nptr)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1))) ;
__extension__ extern long long int atoll (const char *__nptr)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1))) ;
extern double strtod (const char *__restrict __nptr,
char **__restrict __endptr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern float strtof (const char *__restrict __nptr,
char **__restrict __endptr) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern long double strtold (const char *__restrict __nptr,
char **__restrict __endptr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern _Float32 strtof32 (const char *__restrict __nptr,
char **__restrict __endptr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern _Float64 strtof64 (const char *__restrict __nptr,
char **__restrict __endptr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern _Float128 strtof128 (const char *__restrict __nptr,
char **__restrict __endptr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern _Float32x strtof32x (const char *__restrict __nptr,
char **__restrict __endptr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern _Float64x strtof64x (const char *__restrict __nptr,
char **__restrict __endptr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern long int strtol (const char *__restrict __nptr,
char **__restrict __endptr, int __base)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern unsigned long int strtoul (const char *__restrict __nptr,
char **__restrict __endptr, int __base)
noexcept (true) __attribute__ ((__nonnull__ (1)));
__extension__
extern long long int strtoq (const char *__restrict __nptr,
char **__restrict __endptr, int __base)
noexcept (true) __attribute__ ((__nonnull__ (1)));
__extension__
extern unsigned long long int strtouq (const char *__restrict __nptr,
char **__restrict __endptr, int __base)
noexcept (true) __attribute__ ((__nonnull__ (1)));
__extension__
extern long long int strtoll (const char *__restrict __nptr,
char **__restrict __endptr, int __base)
noexcept (true) __attribute__ ((__nonnull__ (1)));
__extension__
extern unsigned long long int strtoull (const char *__restrict __nptr,
char **__restrict __endptr, int __base)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int strfromd (char *__dest, size_t __size, const char *__format,
double __f)
noexcept (true) __attribute__ ((__nonnull__ (3)));
extern int strfromf (char *__dest, size_t __size, const char *__format,
float __f)
noexcept (true) __attribute__ ((__nonnull__ (3)));
extern int strfroml (char *__dest, size_t __size, const char *__format,
long double __f)
noexcept (true) __attribute__ ((__nonnull__ (3)));
extern int strfromf32 (char *__dest, size_t __size, const char * __format,
_Float32 __f)
noexcept (true) __attribute__ ((__nonnull__ (3)));
extern int strfromf64 (char *__dest, size_t __size, const char * __format,
_Float64 __f)
noexcept (true) __attribute__ ((__nonnull__ (3)));
extern int strfromf128 (char *__dest, size_t __size, const char * __format,
_Float128 __f)
noexcept (true) __attribute__ ((__nonnull__ (3)));
extern int strfromf32x (char *__dest, size_t __size, const char * __format,
_Float32x __f)
noexcept (true) __attribute__ ((__nonnull__ (3)));
extern int strfromf64x (char *__dest, size_t __size, const char * __format,
_Float64x __f)
noexcept (true) __attribute__ ((__nonnull__ (3)));
struct __locale_struct
{
struct __locale_data *__locales[13];
const unsigned short int *__ctype_b;
const int *__ctype_tolower;
const int *__ctype_toupper;
const char *__names[13];
};
typedef struct __locale_struct *__locale_t;
typedef __locale_t locale_t;
extern long int strtol_l (const char *__restrict __nptr,
char **__restrict __endptr, int __base,
locale_t __loc) noexcept (true) __attribute__ ((__nonnull__ (1, 4)));
extern unsigned long int strtoul_l (const char *__restrict __nptr,
char **__restrict __endptr,
int __base, locale_t __loc)
noexcept (true) __attribute__ ((__nonnull__ (1, 4)));
__extension__
extern long long int strtoll_l (const char *__restrict __nptr,
char **__restrict __endptr, int __base,
locale_t __loc)
noexcept (true) __attribute__ ((__nonnull__ (1, 4)));
__extension__
extern unsigned long long int strtoull_l (const char *__restrict __nptr,
char **__restrict __endptr,
int __base, locale_t __loc)
noexcept (true) __attribute__ ((__nonnull__ (1, 4)));
extern double strtod_l (const char *__restrict __nptr,
char **__restrict __endptr, locale_t __loc)
noexcept (true) __attribute__ ((__nonnull__ (1, 3)));
extern float strtof_l (const char *__restrict __nptr,
char **__restrict __endptr, locale_t __loc)
noexcept (true) __attribute__ ((__nonnull__ (1, 3)));
extern long double strtold_l (const char *__restrict __nptr,
char **__restrict __endptr,
locale_t __loc)
noexcept (true) __attribute__ ((__nonnull__ (1, 3)));
extern _Float32 strtof32_l (const char *__restrict __nptr,
char **__restrict __endptr,
locale_t __loc)
noexcept (true) __attribute__ ((__nonnull__ (1, 3)));
extern _Float64 strtof64_l (const char *__restrict __nptr,
char **__restrict __endptr,
locale_t __loc)
noexcept (true) __attribute__ ((__nonnull__ (1, 3)));
extern _Float128 strtof128_l (const char *__restrict __nptr,
char **__restrict __endptr,
locale_t __loc)
noexcept (true) __attribute__ ((__nonnull__ (1, 3)));
extern _Float32x strtof32x_l (const char *__restrict __nptr,
char **__restrict __endptr,
locale_t __loc)
noexcept (true) __attribute__ ((__nonnull__ (1, 3)));
extern _Float64x strtof64x_l (const char *__restrict __nptr,
char **__restrict __endptr,
locale_t __loc)
noexcept (true) __attribute__ ((__nonnull__ (1, 3)));
extern __inline __attribute__ ((__gnu_inline__)) int
__attribute__ ((__leaf__)) atoi (const char *__nptr) noexcept (true)
{
return (int) strtol (__nptr, (char **) __null, 10);
}
extern __inline __attribute__ ((__gnu_inline__)) long int
__attribute__ ((__leaf__)) atol (const char *__nptr) noexcept (true)
{
return strtol (__nptr, (char **) __null, 10);
}
__extension__ extern __inline __attribute__ ((__gnu_inline__)) long long int
__attribute__ ((__leaf__)) atoll (const char *__nptr) noexcept (true)
{
return strtoll (__nptr, (char **) __null, 10);
}
extern char *l64a (long int __n) noexcept (true) ;
extern long int a64l (const char *__s)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1))) ;
extern "C" {
typedef __u_char u_char;
typedef __u_short u_short;
typedef __u_int u_int;
typedef __u_long u_long;
typedef __quad_t quad_t;
typedef __u_quad_t u_quad_t;
typedef __fsid_t fsid_t;
typedef __loff_t loff_t;
typedef __ino_t ino_t;
typedef __ino64_t ino64_t;
typedef __dev_t dev_t;
typedef __gid_t gid_t;
typedef __mode_t mode_t;
typedef __nlink_t nlink_t;
typedef __uid_t uid_t;
typedef __pid_t pid_t;
typedef __id_t id_t;
typedef __daddr_t daddr_t;
typedef __caddr_t caddr_t;
typedef __key_t key_t;
typedef __clock_t clock_t;
typedef __clockid_t clockid_t;
typedef __time_t time_t;
typedef __timer_t timer_t;
typedef __useconds_t useconds_t;
typedef __suseconds_t suseconds_t;
typedef unsigned long int ulong;
typedef unsigned short int ushort;
typedef unsigned int uint;
typedef __int8_t int8_t;
typedef __int16_t int16_t;
typedef __int32_t int32_t;
typedef __int64_t int64_t;
typedef __uint8_t u_int8_t;
typedef __uint16_t u_int16_t;
typedef __uint32_t u_int32_t;
typedef __uint64_t u_int64_t;
typedef int register_t __attribute__ ((__mode__ (__word__)));
static __inline __uint16_t
__bswap_16 (__uint16_t __bsx)
{
return __builtin_bswap16 (__bsx);
}
static __inline __uint32_t
__bswap_32 (__uint32_t __bsx)
{
return __builtin_bswap32 (__bsx);
}
__extension__ static __inline __uint64_t
__bswap_64 (__uint64_t __bsx)
{
return __builtin_bswap64 (__bsx);
}
static __inline __uint16_t
__uint16_identity (__uint16_t __x)
{
return __x;
}
static __inline __uint32_t
__uint32_identity (__uint32_t __x)
{
return __x;
}
static __inline __uint64_t
__uint64_identity (__uint64_t __x)
{
return __x;
}
typedef struct
{
unsigned long int __val[(1024 / (8 * sizeof (unsigned long int)))];
} __sigset_t;
typedef __sigset_t sigset_t;
struct timeval
{
__time_t tv_sec;
__suseconds_t tv_usec;
};
struct timespec
{
__time_t tv_sec;
__syscall_slong_t tv_nsec;
};
typedef long int __fd_mask;
typedef struct
{
__fd_mask fds_bits[1024 / (8 * (int) sizeof (__fd_mask))];
} fd_set;
typedef __fd_mask fd_mask;
extern "C" {
extern int select (int __nfds, fd_set *__restrict __readfds,
fd_set *__restrict __writefds,
fd_set *__restrict __exceptfds,
struct timeval *__restrict __timeout);
extern int pselect (int __nfds, fd_set *__restrict __readfds,
fd_set *__restrict __writefds,
fd_set *__restrict __exceptfds,
const struct timespec *__restrict __timeout,
const __sigset_t *__restrict __sigmask);
}
typedef __blksize_t blksize_t;
typedef __blkcnt_t blkcnt_t;
typedef __fsblkcnt_t fsblkcnt_t;
typedef __fsfilcnt_t fsfilcnt_t;
typedef __blkcnt64_t blkcnt64_t;
typedef __fsblkcnt64_t fsblkcnt64_t;
typedef __fsfilcnt64_t fsfilcnt64_t;
typedef union
{
__extension__ unsigned long long int __value64;
struct
{
unsigned int __low;
unsigned int __high;
} __value32;
} __atomic_wide_counter;
typedef struct __pthread_internal_list
{
struct __pthread_internal_list *__prev;
struct __pthread_internal_list *__next;
} __pthread_list_t;
typedef struct __pthread_internal_slist
{
struct __pthread_internal_slist *__next;
} __pthread_slist_t;
struct __pthread_mutex_s
{
int __lock;
unsigned int __count;
int __owner;
int __kind;
unsigned int __nusers;
__extension__ union
{
struct
{
short __espins;
short __eelision;
} __elision_data;
__pthread_slist_t __list;
};
};
struct __pthread_rwlock_arch_t
{
unsigned int __readers;
unsigned int __writers;
unsigned int __wrphase_futex;
unsigned int __writers_futex;
unsigned int __pad3;
unsigned int __pad4;
unsigned char __flags;
unsigned char __shared;
signed char __rwelision;
unsigned char __pad2;
int __cur_writer;
};
struct __pthread_cond_s
{
__atomic_wide_counter __wseq;
__atomic_wide_counter __g1_start;
unsigned int __g_refs[2] ;
unsigned int __g_size[2];
unsigned int __g1_orig_size;
unsigned int __wrefs;
unsigned int __g_signals[2];
};
typedef unsigned int __tss_t;
typedef unsigned long int __thrd_t;
typedef struct
{
int __data ;
} __once_flag;
typedef unsigned long int pthread_t;
typedef union
{
char __size[4];
int __align;
} pthread_mutexattr_t;
typedef union
{
char __size[4];
int __align;
} pthread_condattr_t;
typedef unsigned int pthread_key_t;
typedef int pthread_once_t;
union pthread_attr_t
{
char __size[36];
long int __align;
};
typedef union pthread_attr_t pthread_attr_t;
typedef union
{
struct __pthread_mutex_s __data;
char __size[24];
long int __align;
} pthread_mutex_t;
typedef union
{
struct __pthread_cond_s __data;
char __size[48];
__extension__ long long int __align;
} pthread_cond_t;
typedef union
{
struct __pthread_rwlock_arch_t __data;
char __size[32];
long int __align;
} pthread_rwlock_t;
typedef union
{
char __size[8];
long int __align;
} pthread_rwlockattr_t;
typedef volatile int pthread_spinlock_t;
typedef union
{
char __size[20];
long int __align;
} pthread_barrier_t;
typedef union
{
char __size[4];
int __align;
} pthread_barrierattr_t;
}
extern long int random (void) noexcept (true);
extern void srandom (unsigned int __seed) noexcept (true);
extern char *initstate (unsigned int __seed, char *__statebuf,
size_t __statelen) noexcept (true) __attribute__ ((__nonnull__ (2)));
extern char *setstate (char *__statebuf) noexcept (true) __attribute__ ((__nonnull__ (1)));
struct random_data
{
int32_t *fptr;
int32_t *rptr;
int32_t *state;
int rand_type;
int rand_deg;
int rand_sep;
int32_t *end_ptr;
};
extern int random_r (struct random_data *__restrict __buf,
int32_t *__restrict __result) noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int srandom_r (unsigned int __seed, struct random_data *__buf)
noexcept (true) __attribute__ ((__nonnull__ (2)));
extern int initstate_r (unsigned int __seed, char *__restrict __statebuf,
size_t __statelen,
struct random_data *__restrict __buf)
noexcept (true) __attribute__ ((__nonnull__ (2, 4)));
extern int setstate_r (char *__restrict __statebuf,
struct random_data *__restrict __buf)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int rand (void) noexcept (true);
extern void srand (unsigned int __seed) noexcept (true);
extern int rand_r (unsigned int *__seed) noexcept (true);
extern double drand48 (void) noexcept (true);
extern double erand48 (unsigned short int __xsubi[3]) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern long int lrand48 (void) noexcept (true);
extern long int nrand48 (unsigned short int __xsubi[3])
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern long int mrand48 (void) noexcept (true);
extern long int jrand48 (unsigned short int __xsubi[3])
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern void srand48 (long int __seedval) noexcept (true);
extern unsigned short int *seed48 (unsigned short int __seed16v[3])
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern void lcong48 (unsigned short int __param[7]) noexcept (true) __attribute__ ((__nonnull__ (1)));
struct drand48_data
{
unsigned short int __x[3];
unsigned short int __old_x[3];
unsigned short int __c;
unsigned short int __init;
__extension__ unsigned long long int __a;
};
extern int drand48_r (struct drand48_data *__restrict __buffer,
double *__restrict __result) noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int erand48_r (unsigned short int __xsubi[3],
struct drand48_data *__restrict __buffer,
double *__restrict __result) noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int lrand48_r (struct drand48_data *__restrict __buffer,
long int *__restrict __result)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int nrand48_r (unsigned short int __xsubi[3],
struct drand48_data *__restrict __buffer,
long int *__restrict __result)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int mrand48_r (struct drand48_data *__restrict __buffer,
long int *__restrict __result)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int jrand48_r (unsigned short int __xsubi[3],
struct drand48_data *__restrict __buffer,
long int *__restrict __result)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int srand48_r (long int __seedval, struct drand48_data *__buffer)
noexcept (true) __attribute__ ((__nonnull__ (2)));
extern int seed48_r (unsigned short int __seed16v[3],
struct drand48_data *__buffer) noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int lcong48_r (unsigned short int __param[7],
struct drand48_data *__buffer)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern __uint32_t arc4random (void)
noexcept (true) ;
extern void arc4random_buf (void *__buf, size_t __size)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern __uint32_t arc4random_uniform (__uint32_t __upper_bound)
noexcept (true) ;
extern void *malloc (size_t __size) noexcept (true) __attribute__ ((__malloc__))
__attribute__ ((__alloc_size__ (1))) ;
extern void *calloc (size_t __nmemb, size_t __size)
noexcept (true) __attribute__ ((__malloc__)) __attribute__ ((__alloc_size__ (1, 2))) ;
extern void *realloc (void *__ptr, size_t __size)
noexcept (true) __attribute__ ((__warn_unused_result__)) __attribute__ ((__alloc_size__ (2)));
extern void free (void *__ptr) noexcept (true);
extern void *reallocarray (void *__ptr, size_t __nmemb, size_t __size)
noexcept (true) __attribute__ ((__warn_unused_result__))
__attribute__ ((__alloc_size__ (2, 3)))
__attribute__ ((__malloc__ (__builtin_free, 1)));
extern void *reallocarray (void *__ptr, size_t __nmemb, size_t __size)
noexcept (true) __attribute__ ((__malloc__ (reallocarray, 1)));
extern "C" {
extern void *alloca (size_t __size) noexcept (true);
}
extern void *valloc (size_t __size) noexcept (true) __attribute__ ((__malloc__))
__attribute__ ((__alloc_size__ (1))) ;
extern int posix_memalign (void **__memptr, size_t __alignment, size_t __size)
noexcept (true) __attribute__ ((__nonnull__ (1))) ;
extern void *aligned_alloc (size_t __alignment, size_t __size)
noexcept (true) __attribute__ ((__malloc__)) __attribute__ ((__alloc_align__ (1)))
__attribute__ ((__alloc_size__ (2))) ;
extern void abort (void) noexcept (true) __attribute__ ((__noreturn__));
extern int atexit (void (*__func) (void)) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern "C++" int at_quick_exit (void (*__func) (void))
noexcept (true) __asm ("at_quick_exit") __attribute__ ((__nonnull__ (1)));
extern int on_exit (void (*__func) (int __status, void *__arg), void *__arg)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern void exit (int __status) noexcept (true) __attribute__ ((__noreturn__));
extern void quick_exit (int __status) noexcept (true) __attribute__ ((__noreturn__));
extern void _Exit (int __status) noexcept (true) __attribute__ ((__noreturn__));
extern char *getenv (const char *__name) noexcept (true) __attribute__ ((__nonnull__ (1))) ;
extern char *secure_getenv (const char *__name)
noexcept (true) __attribute__ ((__nonnull__ (1))) ;
extern int putenv (char *__string) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int setenv (const char *__name, const char *__value, int __replace)
noexcept (true) __attribute__ ((__nonnull__ (2)));
extern int unsetenv (const char *__name) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int clearenv (void) noexcept (true);
extern char *mktemp (char *__template) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int mkstemp (char *__template) __attribute__ ((__nonnull__ (1))) ;
extern int mkstemp64 (char *__template) __attribute__ ((__nonnull__ (1))) ;
extern int mkstemps (char *__template, int __suffixlen) __attribute__ ((__nonnull__ (1))) ;
extern int mkstemps64 (char *__template, int __suffixlen)
__attribute__ ((__nonnull__ (1))) ;
extern char *mkdtemp (char *__template) noexcept (true) __attribute__ ((__nonnull__ (1))) ;
extern int mkostemp (char *__template, int __flags) __attribute__ ((__nonnull__ (1))) ;
extern int mkostemp64 (char *__template, int __flags) __attribute__ ((__nonnull__ (1))) ;
extern int mkostemps (char *__template, int __suffixlen, int __flags)
__attribute__ ((__nonnull__ (1))) ;
extern int mkostemps64 (char *__template, int __suffixlen, int __flags)
__attribute__ ((__nonnull__ (1))) ;
extern int system (const char *__command) ;
extern char *canonicalize_file_name (const char *__name)
noexcept (true) __attribute__ ((__nonnull__ (1))) __attribute__ ((__malloc__))
__attribute__ ((__malloc__ (__builtin_free, 1))) ;
extern char *realpath (const char *__restrict __name,
char *__restrict __resolved) noexcept (true) ;
typedef int (*__compar_fn_t) (const void *, const void *);
typedef __compar_fn_t comparison_fn_t;
typedef int (*__compar_d_fn_t) (const void *, const void *, void *);
extern void *bsearch (const void *__key, const void *__base,
size_t __nmemb, size_t __size, __compar_fn_t __compar)
__attribute__ ((__nonnull__ (1, 2, 5))) ;
extern __inline __attribute__ ((__gnu_inline__)) void *
bsearch (const void *__key, const void *__base, size_t __nmemb, size_t __size,
__compar_fn_t __compar)
{
size_t __l, __u, __idx;
const void *__p;
int __comparison;
__l = 0;
__u = __nmemb;
while (__l < __u)
{
__idx = (__l + __u) / 2;
__p = (const void *) (((const char *) __base) + (__idx * __size));
__comparison = (*__compar) (__key, __p);
if (__comparison < 0)
__u = __idx;
else if (__comparison > 0)
__l = __idx + 1;
else
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wcast-qual"
return (void *) __p;
#pragma GCC diagnostic pop
}
}
return __null;
}
extern void qsort (void *__base, size_t __nmemb, size_t __size,
__compar_fn_t __compar) __attribute__ ((__nonnull__ (1, 4)));
extern void qsort_r (void *__base, size_t __nmemb, size_t __size,
__compar_d_fn_t __compar, void *__arg)
__attribute__ ((__nonnull__ (1, 4)));
extern int abs (int __x) noexcept (true) __attribute__ ((__const__)) ;
extern long int labs (long int __x) noexcept (true) __attribute__ ((__const__)) ;
__extension__ extern long long int llabs (long long int __x)
noexcept (true) __attribute__ ((__const__)) ;
extern div_t div (int __numer, int __denom)
noexcept (true) __attribute__ ((__const__)) ;
extern ldiv_t ldiv (long int __numer, long int __denom)
noexcept (true) __attribute__ ((__const__)) ;
__extension__ extern lldiv_t lldiv (long long int __numer,
long long int __denom)
noexcept (true) __attribute__ ((__const__)) ;
extern char *ecvt (double __value, int __ndigit, int *__restrict __decpt,
int *__restrict __sign) noexcept (true) __attribute__ ((__nonnull__ (3, 4))) ;
extern char *fcvt (double __value, int __ndigit, int *__restrict __decpt,
int *__restrict __sign) noexcept (true) __attribute__ ((__nonnull__ (3, 4))) ;
extern char *gcvt (double __value, int __ndigit, char *__buf)
noexcept (true) __attribute__ ((__nonnull__ (3))) ;
extern char *qecvt (long double __value, int __ndigit,
int *__restrict __decpt, int *__restrict __sign)
noexcept (true) __attribute__ ((__nonnull__ (3, 4))) ;
extern char *qfcvt (long double __value, int __ndigit,
int *__restrict __decpt, int *__restrict __sign)
noexcept (true) __attribute__ ((__nonnull__ (3, 4))) ;
extern char *qgcvt (long double __value, int __ndigit, char *__buf)
noexcept (true) __attribute__ ((__nonnull__ (3))) ;
extern int ecvt_r (double __value, int __ndigit, int *__restrict __decpt,
int *__restrict __sign, char *__restrict __buf,
size_t __len) noexcept (true) __attribute__ ((__nonnull__ (3, 4, 5)));
extern int fcvt_r (double __value, int __ndigit, int *__restrict __decpt,
int *__restrict __sign, char *__restrict __buf,
size_t __len) noexcept (true) __attribute__ ((__nonnull__ (3, 4, 5)));
extern int qecvt_r (long double __value, int __ndigit,
int *__restrict __decpt, int *__restrict __sign,
char *__restrict __buf, size_t __len)
noexcept (true) __attribute__ ((__nonnull__ (3, 4, 5)));
extern int qfcvt_r (long double __value, int __ndigit,
int *__restrict __decpt, int *__restrict __sign,
char *__restrict __buf, size_t __len)
noexcept (true) __attribute__ ((__nonnull__ (3, 4, 5)));
extern int mblen (const char *__s, size_t __n) noexcept (true);
extern int mbtowc (wchar_t *__restrict __pwc,
const char *__restrict __s, size_t __n) noexcept (true);
extern int wctomb (char *__s, wchar_t __wchar) noexcept (true);
extern size_t mbstowcs (wchar_t *__restrict __pwcs,
const char *__restrict __s, size_t __n) noexcept (true)
__attribute__ ((__access__ (__read_only__, 2)));
extern size_t wcstombs (char *__restrict __s,
const wchar_t *__restrict __pwcs, size_t __n)
noexcept (true)
__attribute__ ((__access__ (__write_only__, 1, 3)))
__attribute__ ((__access__ (__read_only__, 2)));
extern int rpmatch (const char *__response) noexcept (true) __attribute__ ((__nonnull__ (1))) ;
extern int getsubopt (char **__restrict __optionp,
char *const *__restrict __tokens,
char **__restrict __valuep)
noexcept (true) __attribute__ ((__nonnull__ (1, 2, 3))) ;
extern int posix_openpt (int __oflag) ;
extern int grantpt (int __fd) noexcept (true);
extern int unlockpt (int __fd) noexcept (true);
extern char *ptsname (int __fd) noexcept (true) ;
extern int ptsname_r (int __fd, char *__buf, size_t __buflen)
noexcept (true) __attribute__ ((__nonnull__ (2))) __attribute__ ((__access__ (__write_only__, 2, 3)));
extern int getpt (void);
extern int getloadavg (double __loadavg[], int __nelem)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern __inline __attribute__ ((__gnu_inline__)) double
__attribute__ ((__leaf__)) atof (const char *__nptr) noexcept (true)
{
return strtod (__nptr, (char **) __null);
}
}
extern "C++"
{
namespace std __attribute__ ((__visibility__ ("default")))
{
using ::abs;
inline long
abs(long __i) { return __builtin_labs(__i); }
inline long long
abs(long long __x) { return __builtin_llabs (__x); }
inline constexpr double
abs(double __x)
{ return __builtin_fabs(__x); }
inline constexpr float
abs(float __x)
{ return __builtin_fabsf(__x); }
inline constexpr long double
abs(long double __x)
{ return __builtin_fabsl(__x); }
__extension__ inline constexpr
__float128
abs(__float128 __x)
{
return __builtin_fabsf128(__x);
}
}
}
extern "C++"
{
namespace std __attribute__ ((__visibility__ ("default")))
{
using ::acos;
inline constexpr float
acos(float __x)
{ return __builtin_acosf(__x); }
inline constexpr long double
acos(long double __x)
{ return __builtin_acosl(__x); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
acos(_Tp __x)
{ return __builtin_acos(__x); }
using ::asin;
inline constexpr float
asin(float __x)
{ return __builtin_asinf(__x); }
inline constexpr long double
asin(long double __x)
{ return __builtin_asinl(__x); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
asin(_Tp __x)
{ return __builtin_asin(__x); }
using ::atan;
inline constexpr float
atan(float __x)
{ return __builtin_atanf(__x); }
inline constexpr long double
atan(long double __x)
{ return __builtin_atanl(__x); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
atan(_Tp __x)
{ return __builtin_atan(__x); }
using ::atan2;
inline constexpr float
atan2(float __y, float __x)
{ return __builtin_atan2f(__y, __x); }
inline constexpr long double
atan2(long double __y, long double __x)
{ return __builtin_atan2l(__y, __x); }
using ::ceil;
inline constexpr float
ceil(float __x)
{ return __builtin_ceilf(__x); }
inline constexpr long double
ceil(long double __x)
{ return __builtin_ceill(__x); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
ceil(_Tp __x)
{ return __builtin_ceil(__x); }
using ::cos;
inline constexpr float
cos(float __x)
{ return __builtin_cosf(__x); }
inline constexpr long double
cos(long double __x)
{ return __builtin_cosl(__x); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
cos(_Tp __x)
{ return __builtin_cos(__x); }
using ::cosh;
inline constexpr float
cosh(float __x)
{ return __builtin_coshf(__x); }
inline constexpr long double
cosh(long double __x)
{ return __builtin_coshl(__x); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
cosh(_Tp __x)
{ return __builtin_cosh(__x); }
using ::exp;
inline constexpr float
exp(float __x)
{ return __builtin_expf(__x); }
inline constexpr long double
exp(long double __x)
{ return __builtin_expl(__x); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
exp(_Tp __x)
{ return __builtin_exp(__x); }
using ::fabs;
inline constexpr float
fabs(float __x)
{ return __builtin_fabsf(__x); }
inline constexpr long double
fabs(long double __x)
{ return __builtin_fabsl(__x); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
fabs(_Tp __x)
{ return __builtin_fabs(__x); }
using ::floor;
inline constexpr float
floor(float __x)
{ return __builtin_floorf(__x); }
inline constexpr long double
floor(long double __x)
{ return __builtin_floorl(__x); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
floor(_Tp __x)
{ return __builtin_floor(__x); }
using ::fmod;
inline constexpr float
fmod(float __x, float __y)
{ return __builtin_fmodf(__x, __y); }
inline constexpr long double
fmod(long double __x, long double __y)
{ return __builtin_fmodl(__x, __y); }
using ::frexp;
inline float
frexp(float __x, int* __exp)
{ return __builtin_frexpf(__x, __exp); }
inline long double
frexp(long double __x, int* __exp)
{ return __builtin_frexpl(__x, __exp); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
frexp(_Tp __x, int* __exp)
{ return __builtin_frexp(__x, __exp); }
using ::ldexp;
inline constexpr float
ldexp(float __x, int __exp)
{ return __builtin_ldexpf(__x, __exp); }
inline constexpr long double
ldexp(long double __x, int __exp)
{ return __builtin_ldexpl(__x, __exp); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
ldexp(_Tp __x, int __exp)
{ return __builtin_ldexp(__x, __exp); }
using ::log;
inline constexpr float
log(float __x)
{ return __builtin_logf(__x); }
inline constexpr long double
log(long double __x)
{ return __builtin_logl(__x); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
log(_Tp __x)
{ return __builtin_log(__x); }
using ::log10;
inline constexpr float
log10(float __x)
{ return __builtin_log10f(__x); }
inline constexpr long double
log10(long double __x)
{ return __builtin_log10l(__x); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
log10(_Tp __x)
{ return __builtin_log10(__x); }
using ::modf;
inline float
modf(float __x, float* __iptr)
{ return __builtin_modff(__x, __iptr); }
inline long double
modf(long double __x, long double* __iptr)
{ return __builtin_modfl(__x, __iptr); }
using ::pow;
inline constexpr float
pow(float __x, float __y)
{ return __builtin_powf(__x, __y); }
inline constexpr long double
pow(long double __x, long double __y)
{ return __builtin_powl(__x, __y); }
using ::sin;
inline constexpr float
sin(float __x)
{ return __builtin_sinf(__x); }
inline constexpr long double
sin(long double __x)
{ return __builtin_sinl(__x); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
sin(_Tp __x)
{ return __builtin_sin(__x); }
using ::sinh;
inline constexpr float
sinh(float __x)
{ return __builtin_sinhf(__x); }
inline constexpr long double
sinh(long double __x)
{ return __builtin_sinhl(__x); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
sinh(_Tp __x)
{ return __builtin_sinh(__x); }
using ::sqrt;
inline constexpr float
sqrt(float __x)
{ return __builtin_sqrtf(__x); }
inline constexpr long double
sqrt(long double __x)
{ return __builtin_sqrtl(__x); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
sqrt(_Tp __x)
{ return __builtin_sqrt(__x); }
using ::tan;
inline constexpr float
tan(float __x)
{ return __builtin_tanf(__x); }
inline constexpr long double
tan(long double __x)
{ return __builtin_tanl(__x); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
tan(_Tp __x)
{ return __builtin_tan(__x); }
using ::tanh;
inline constexpr float
tanh(float __x)
{ return __builtin_tanhf(__x); }
inline constexpr long double
tanh(long double __x)
{ return __builtin_tanhl(__x); }
template<typename _Tp>
inline constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
tanh(_Tp __x)
{ return __builtin_tanh(__x); }
template<typename _Tp, typename _Up>
inline constexpr
typename __gnu_cxx::__promote_2<_Tp, _Up>::__type
atan2(_Tp __y, _Up __x)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Up>::__type __type;
return atan2(__type(__y), __type(__x));
}
template<typename _Tp, typename _Up>
inline constexpr
typename __gnu_cxx::__promote_2<_Tp, _Up>::__type
fmod(_Tp __x, _Up __y)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Up>::__type __type;
return fmod(__type(__x), __type(__y));
}
template<typename _Tp, typename _Up>
inline constexpr
typename __gnu_cxx::__promote_2<_Tp, _Up>::__type
pow(_Tp __x, _Up __y)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Up>::__type __type;
return pow(__type(__x), __type(__y));
}
constexpr int
fpclassify(float __x)
{ return __builtin_fpclassify(0, 1, 4,
3, 2, __x); }
constexpr int
fpclassify(double __x)
{ return __builtin_fpclassify(0, 1, 4,
3, 2, __x); }
constexpr int
fpclassify(long double __x)
{ return __builtin_fpclassify(0, 1, 4,
3, 2, __x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
int>::__type
fpclassify(_Tp __x)
{ return __x != 0 ? 4 : 2; }
constexpr bool
isfinite(float __x)
{ return __builtin_isfinite(__x); }
constexpr bool
isfinite(double __x)
{ return __builtin_isfinite(__x); }
constexpr bool
isfinite(long double __x)
{ return __builtin_isfinite(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
bool>::__type
isfinite(_Tp __x)
{ return true; }
constexpr bool
isinf(float __x)
{ return __builtin_isinf(__x); }
constexpr bool
isinf(double __x)
{ return __builtin_isinf(__x); }
constexpr bool
isinf(long double __x)
{ return __builtin_isinf(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
bool>::__type
isinf(_Tp __x)
{ return false; }
constexpr bool
isnan(float __x)
{ return __builtin_isnan(__x); }
constexpr bool
isnan(double __x)
{ return __builtin_isnan(__x); }
constexpr bool
isnan(long double __x)
{ return __builtin_isnan(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
bool>::__type
isnan(_Tp __x)
{ return false; }
constexpr bool
isnormal(float __x)
{ return __builtin_isnormal(__x); }
constexpr bool
isnormal(double __x)
{ return __builtin_isnormal(__x); }
constexpr bool
isnormal(long double __x)
{ return __builtin_isnormal(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
bool>::__type
isnormal(_Tp __x)
{ return __x != 0 ? true : false; }
constexpr bool
signbit(float __x)
{ return __builtin_signbit(__x); }
constexpr bool
signbit(double __x)
{ return __builtin_signbit(__x); }
constexpr bool
signbit(long double __x)
{ return __builtin_signbit(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
bool>::__type
signbit(_Tp __x)
{ return __x < 0 ? true : false; }
constexpr bool
isgreater(float __x, float __y)
{ return __builtin_isgreater(__x, __y); }
constexpr bool
isgreater(double __x, double __y)
{ return __builtin_isgreater(__x, __y); }
constexpr bool
isgreater(long double __x, long double __y)
{ return __builtin_isgreater(__x, __y); }
template<typename _Tp, typename _Up>
constexpr typename
__gnu_cxx::__enable_if<(__is_arithmetic<_Tp>::__value
&& __is_arithmetic<_Up>::__value), bool>::__type
isgreater(_Tp __x, _Up __y)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Up>::__type __type;
return __builtin_isgreater(__type(__x), __type(__y));
}
constexpr bool
isgreaterequal(float __x, float __y)
{ return __builtin_isgreaterequal(__x, __y); }
constexpr bool
isgreaterequal(double __x, double __y)
{ return __builtin_isgreaterequal(__x, __y); }
constexpr bool
isgreaterequal(long double __x, long double __y)
{ return __builtin_isgreaterequal(__x, __y); }
template<typename _Tp, typename _Up>
constexpr typename
__gnu_cxx::__enable_if<(__is_arithmetic<_Tp>::__value
&& __is_arithmetic<_Up>::__value), bool>::__type
isgreaterequal(_Tp __x, _Up __y)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Up>::__type __type;
return __builtin_isgreaterequal(__type(__x), __type(__y));
}
constexpr bool
isless(float __x, float __y)
{ return __builtin_isless(__x, __y); }
constexpr bool
isless(double __x, double __y)
{ return __builtin_isless(__x, __y); }
constexpr bool
isless(long double __x, long double __y)
{ return __builtin_isless(__x, __y); }
template<typename _Tp, typename _Up>
constexpr typename
__gnu_cxx::__enable_if<(__is_arithmetic<_Tp>::__value
&& __is_arithmetic<_Up>::__value), bool>::__type
isless(_Tp __x, _Up __y)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Up>::__type __type;
return __builtin_isless(__type(__x), __type(__y));
}
constexpr bool
islessequal(float __x, float __y)
{ return __builtin_islessequal(__x, __y); }
constexpr bool
islessequal(double __x, double __y)
{ return __builtin_islessequal(__x, __y); }
constexpr bool
islessequal(long double __x, long double __y)
{ return __builtin_islessequal(__x, __y); }
template<typename _Tp, typename _Up>
constexpr typename
__gnu_cxx::__enable_if<(__is_arithmetic<_Tp>::__value
&& __is_arithmetic<_Up>::__value), bool>::__type
islessequal(_Tp __x, _Up __y)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Up>::__type __type;
return __builtin_islessequal(__type(__x), __type(__y));
}
constexpr bool
islessgreater(float __x, float __y)
{ return __builtin_islessgreater(__x, __y); }
constexpr bool
islessgreater(double __x, double __y)
{ return __builtin_islessgreater(__x, __y); }
constexpr bool
islessgreater(long double __x, long double __y)
{ return __builtin_islessgreater(__x, __y); }
template<typename _Tp, typename _Up>
constexpr typename
__gnu_cxx::__enable_if<(__is_arithmetic<_Tp>::__value
&& __is_arithmetic<_Up>::__value), bool>::__type
islessgreater(_Tp __x, _Up __y)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Up>::__type __type;
return __builtin_islessgreater(__type(__x), __type(__y));
}
constexpr bool
isunordered(float __x, float __y)
{ return __builtin_isunordered(__x, __y); }
constexpr bool
isunordered(double __x, double __y)
{ return __builtin_isunordered(__x, __y); }
constexpr bool
isunordered(long double __x, long double __y)
{ return __builtin_isunordered(__x, __y); }
template<typename _Tp, typename _Up>
constexpr typename
__gnu_cxx::__enable_if<(__is_arithmetic<_Tp>::__value
&& __is_arithmetic<_Up>::__value), bool>::__type
isunordered(_Tp __x, _Up __y)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Up>::__type __type;
return __builtin_isunordered(__type(__x), __type(__y));
}
using ::double_t;
using ::float_t;
using ::acosh;
using ::acoshf;
using ::acoshl;
using ::asinh;
using ::asinhf;
using ::asinhl;
using ::atanh;
using ::atanhf;
using ::atanhl;
using ::cbrt;
using ::cbrtf;
using ::cbrtl;
using ::copysign;
using ::copysignf;
using ::copysignl;
using ::erf;
using ::erff;
using ::erfl;
using ::erfc;
using ::erfcf;
using ::erfcl;
using ::exp2;
using ::exp2f;
using ::exp2l;
using ::expm1;
using ::expm1f;
using ::expm1l;
using ::fdim;
using ::fdimf;
using ::fdiml;
using ::fma;
using ::fmaf;
using ::fmal;
using ::fmax;
using ::fmaxf;
using ::fmaxl;
using ::fmin;
using ::fminf;
using ::fminl;
using ::hypot;
using ::hypotf;
using ::hypotl;
using ::ilogb;
using ::ilogbf;
using ::ilogbl;
using ::lgamma;
using ::lgammaf;
using ::lgammal;
using ::llrint;
using ::llrintf;
using ::llrintl;
using ::llround;
using ::llroundf;
using ::llroundl;
using ::log1p;
using ::log1pf;
using ::log1pl;
using ::log2;
using ::log2f;
using ::log2l;
using ::logb;
using ::logbf;
using ::logbl;
using ::lrint;
using ::lrintf;
using ::lrintl;
using ::lround;
using ::lroundf;
using ::lroundl;
using ::nan;
using ::nanf;
using ::nanl;
using ::nearbyint;
using ::nearbyintf;
using ::nearbyintl;
using ::nextafter;
using ::nextafterf;
using ::nextafterl;
using ::nexttoward;
using ::nexttowardf;
using ::nexttowardl;
using ::remainder;
using ::remainderf;
using ::remainderl;
using ::remquo;
using ::remquof;
using ::remquol;
using ::rint;
using ::rintf;
using ::rintl;
using ::round;
using ::roundf;
using ::roundl;
using ::scalbln;
using ::scalblnf;
using ::scalblnl;
using ::scalbn;
using ::scalbnf;
using ::scalbnl;
using ::tgamma;
using ::tgammaf;
using ::tgammal;
using ::trunc;
using ::truncf;
using ::truncl;
constexpr float
acosh(float __x)
{ return __builtin_acoshf(__x); }
constexpr long double
acosh(long double __x)
{ return __builtin_acoshl(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
acosh(_Tp __x)
{ return __builtin_acosh(__x); }
constexpr float
asinh(float __x)
{ return __builtin_asinhf(__x); }
constexpr long double
asinh(long double __x)
{ return __builtin_asinhl(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
asinh(_Tp __x)
{ return __builtin_asinh(__x); }
constexpr float
atanh(float __x)
{ return __builtin_atanhf(__x); }
constexpr long double
atanh(long double __x)
{ return __builtin_atanhl(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
atanh(_Tp __x)
{ return __builtin_atanh(__x); }
constexpr float
cbrt(float __x)
{ return __builtin_cbrtf(__x); }
constexpr long double
cbrt(long double __x)
{ return __builtin_cbrtl(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
cbrt(_Tp __x)
{ return __builtin_cbrt(__x); }
constexpr float
copysign(float __x, float __y)
{ return __builtin_copysignf(__x, __y); }
constexpr long double
copysign(long double __x, long double __y)
{ return __builtin_copysignl(__x, __y); }
constexpr float
erf(float __x)
{ return __builtin_erff(__x); }
constexpr long double
erf(long double __x)
{ return __builtin_erfl(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
erf(_Tp __x)
{ return __builtin_erf(__x); }
constexpr float
erfc(float __x)
{ return __builtin_erfcf(__x); }
constexpr long double
erfc(long double __x)
{ return __builtin_erfcl(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
erfc(_Tp __x)
{ return __builtin_erfc(__x); }
constexpr float
exp2(float __x)
{ return __builtin_exp2f(__x); }
constexpr long double
exp2(long double __x)
{ return __builtin_exp2l(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
exp2(_Tp __x)
{ return __builtin_exp2(__x); }
constexpr float
expm1(float __x)
{ return __builtin_expm1f(__x); }
constexpr long double
expm1(long double __x)
{ return __builtin_expm1l(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
expm1(_Tp __x)
{ return __builtin_expm1(__x); }
constexpr float
fdim(float __x, float __y)
{ return __builtin_fdimf(__x, __y); }
constexpr long double
fdim(long double __x, long double __y)
{ return __builtin_fdiml(__x, __y); }
constexpr float
fma(float __x, float __y, float __z)
{ return __builtin_fmaf(__x, __y, __z); }
constexpr long double
fma(long double __x, long double __y, long double __z)
{ return __builtin_fmal(__x, __y, __z); }
constexpr float
fmax(float __x, float __y)
{ return __builtin_fmaxf(__x, __y); }
constexpr long double
fmax(long double __x, long double __y)
{ return __builtin_fmaxl(__x, __y); }
constexpr float
fmin(float __x, float __y)
{ return __builtin_fminf(__x, __y); }
constexpr long double
fmin(long double __x, long double __y)
{ return __builtin_fminl(__x, __y); }
constexpr float
hypot(float __x, float __y)
{ return __builtin_hypotf(__x, __y); }
constexpr long double
hypot(long double __x, long double __y)
{ return __builtin_hypotl(__x, __y); }
constexpr int
ilogb(float __x)
{ return __builtin_ilogbf(__x); }
constexpr int
ilogb(long double __x)
{ return __builtin_ilogbl(__x); }
template<typename _Tp>
constexpr
typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
int>::__type
ilogb(_Tp __x)
{ return __builtin_ilogb(__x); }
constexpr float
lgamma(float __x)
{ return __builtin_lgammaf(__x); }
constexpr long double
lgamma(long double __x)
{ return __builtin_lgammal(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
lgamma(_Tp __x)
{ return __builtin_lgamma(__x); }
constexpr long long
llrint(float __x)
{ return __builtin_llrintf(__x); }
constexpr long long
llrint(long double __x)
{ return __builtin_llrintl(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
long long>::__type
llrint(_Tp __x)
{ return __builtin_llrint(__x); }
constexpr long long
llround(float __x)
{ return __builtin_llroundf(__x); }
constexpr long long
llround(long double __x)
{ return __builtin_llroundl(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
long long>::__type
llround(_Tp __x)
{ return __builtin_llround(__x); }
constexpr float
log1p(float __x)
{ return __builtin_log1pf(__x); }
constexpr long double
log1p(long double __x)
{ return __builtin_log1pl(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
log1p(_Tp __x)
{ return __builtin_log1p(__x); }
constexpr float
log2(float __x)
{ return __builtin_log2f(__x); }
constexpr long double
log2(long double __x)
{ return __builtin_log2l(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
log2(_Tp __x)
{ return __builtin_log2(__x); }
constexpr float
logb(float __x)
{ return __builtin_logbf(__x); }
constexpr long double
logb(long double __x)
{ return __builtin_logbl(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
logb(_Tp __x)
{ return __builtin_logb(__x); }
constexpr long
lrint(float __x)
{ return __builtin_lrintf(__x); }
constexpr long
lrint(long double __x)
{ return __builtin_lrintl(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
long>::__type
lrint(_Tp __x)
{ return __builtin_lrint(__x); }
constexpr long
lround(float __x)
{ return __builtin_lroundf(__x); }
constexpr long
lround(long double __x)
{ return __builtin_lroundl(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
long>::__type
lround(_Tp __x)
{ return __builtin_lround(__x); }
constexpr float
nearbyint(float __x)
{ return __builtin_nearbyintf(__x); }
constexpr long double
nearbyint(long double __x)
{ return __builtin_nearbyintl(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
nearbyint(_Tp __x)
{ return __builtin_nearbyint(__x); }
constexpr float
nextafter(float __x, float __y)
{ return __builtin_nextafterf(__x, __y); }
constexpr long double
nextafter(long double __x, long double __y)
{ return __builtin_nextafterl(__x, __y); }
constexpr float
nexttoward(float __x, long double __y)
{ return __builtin_nexttowardf(__x, __y); }
constexpr long double
nexttoward(long double __x, long double __y)
{ return __builtin_nexttowardl(__x, __y); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
nexttoward(_Tp __x, long double __y)
{ return __builtin_nexttoward(__x, __y); }
constexpr float
remainder(float __x, float __y)
{ return __builtin_remainderf(__x, __y); }
constexpr long double
remainder(long double __x, long double __y)
{ return __builtin_remainderl(__x, __y); }
inline float
remquo(float __x, float __y, int* __pquo)
{ return __builtin_remquof(__x, __y, __pquo); }
inline long double
remquo(long double __x, long double __y, int* __pquo)
{ return __builtin_remquol(__x, __y, __pquo); }
constexpr float
rint(float __x)
{ return __builtin_rintf(__x); }
constexpr long double
rint(long double __x)
{ return __builtin_rintl(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
rint(_Tp __x)
{ return __builtin_rint(__x); }
constexpr float
round(float __x)
{ return __builtin_roundf(__x); }
constexpr long double
round(long double __x)
{ return __builtin_roundl(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
round(_Tp __x)
{ return __builtin_round(__x); }
constexpr float
scalbln(float __x, long __ex)
{ return __builtin_scalblnf(__x, __ex); }
constexpr long double
scalbln(long double __x, long __ex)
{ return __builtin_scalblnl(__x, __ex); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
scalbln(_Tp __x, long __ex)
{ return __builtin_scalbln(__x, __ex); }
constexpr float
scalbn(float __x, int __ex)
{ return __builtin_scalbnf(__x, __ex); }
constexpr long double
scalbn(long double __x, int __ex)
{ return __builtin_scalbnl(__x, __ex); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
scalbn(_Tp __x, int __ex)
{ return __builtin_scalbn(__x, __ex); }
constexpr float
tgamma(float __x)
{ return __builtin_tgammaf(__x); }
constexpr long double
tgamma(long double __x)
{ return __builtin_tgammal(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
tgamma(_Tp __x)
{ return __builtin_tgamma(__x); }
constexpr float
trunc(float __x)
{ return __builtin_truncf(__x); }
constexpr long double
trunc(long double __x)
{ return __builtin_truncl(__x); }
template<typename _Tp>
constexpr typename __gnu_cxx::__enable_if<__is_integer<_Tp>::__value,
double>::__type
trunc(_Tp __x)
{ return __builtin_trunc(__x); }
template<typename _Tp, typename _Up>
constexpr typename __gnu_cxx::__promote_2<_Tp, _Up>::__type
copysign(_Tp __x, _Up __y)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Up>::__type __type;
return copysign(__type(__x), __type(__y));
}
template<typename _Tp, typename _Up>
constexpr typename __gnu_cxx::__promote_2<_Tp, _Up>::__type
fdim(_Tp __x, _Up __y)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Up>::__type __type;
return fdim(__type(__x), __type(__y));
}
template<typename _Tp, typename _Up>
constexpr typename __gnu_cxx::__promote_2<_Tp, _Up>::__type
fmax(_Tp __x, _Up __y)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Up>::__type __type;
return fmax(__type(__x), __type(__y));
}
template<typename _Tp, typename _Up>
constexpr typename __gnu_cxx::__promote_2<_Tp, _Up>::__type
fmin(_Tp __x, _Up __y)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Up>::__type __type;
return fmin(__type(__x), __type(__y));
}
template<typename _Tp, typename _Up>
constexpr typename __gnu_cxx::__promote_2<_Tp, _Up>::__type
hypot(_Tp __x, _Up __y)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Up>::__type __type;
return hypot(__type(__x), __type(__y));
}
template<typename _Tp, typename _Up>
constexpr typename __gnu_cxx::__promote_2<_Tp, _Up>::__type
nextafter(_Tp __x, _Up __y)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Up>::__type __type;
return nextafter(__type(__x), __type(__y));
}
template<typename _Tp, typename _Up>
constexpr typename __gnu_cxx::__promote_2<_Tp, _Up>::__type
remainder(_Tp __x, _Up __y)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Up>::__type __type;
return remainder(__type(__x), __type(__y));
}
template<typename _Tp, typename _Up>
inline typename __gnu_cxx::__promote_2<_Tp, _Up>::__type
remquo(_Tp __x, _Up __y, int* __pquo)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Up>::__type __type;
return remquo(__type(__x), __type(__y), __pquo);
}
template<typename _Tp, typename _Up, typename _Vp>
constexpr typename __gnu_cxx::__promote_3<_Tp, _Up, _Vp>::__type
fma(_Tp __x, _Up __y, _Vp __z)
{
typedef typename __gnu_cxx::__promote_3<_Tp, _Up, _Vp>::__type __type;
return fma(__type(__x), __type(__y), __type(__z));
}
template<typename _Tp>
inline _Tp
__hypot3(_Tp __x, _Tp __y, _Tp __z)
{
__x = std::abs(__x);
__y = std::abs(__y);
__z = std::abs(__z);
if (_Tp __a = __x < __y ? __y < __z ? __z : __y : __x < __z ? __z : __x)
return __a * std::sqrt((__x / __a) * (__x / __a)
+ (__y / __a) * (__y / __a)
+ (__z / __a) * (__z / __a));
else
return {};
}
inline float
hypot(float __x, float __y, float __z)
{ return std::__hypot3<float>(__x, __y, __z); }
inline double
hypot(double __x, double __y, double __z)
{ return std::__hypot3<double>(__x, __y, __z); }
inline long double
hypot(long double __x, long double __y, long double __z)
{ return std::__hypot3<long double>(__x, __y, __z); }
template<typename _Tp, typename _Up, typename _Vp>
__gnu_cxx::__promoted_t<_Tp, _Up, _Vp>
hypot(_Tp __x, _Up __y, _Vp __z)
{
using __type = __gnu_cxx::__promoted_t<_Tp, _Up, _Vp>;
return std::__hypot3<__type>(__x, __y, __z);
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
void
__throw_bad_exception(void) __attribute__((__noreturn__));
void
__throw_bad_alloc(void) __attribute__((__noreturn__));
void
__throw_bad_array_new_length(void) __attribute__((__noreturn__));
void
__throw_bad_cast(void) __attribute__((__noreturn__));
void
__throw_bad_typeid(void) __attribute__((__noreturn__));
void
__throw_logic_error(const char*) __attribute__((__noreturn__));
void
__throw_domain_error(const char*) __attribute__((__noreturn__));
void
__throw_invalid_argument(const char*) __attribute__((__noreturn__));
void
__throw_length_error(const char*) __attribute__((__noreturn__));
void
__throw_out_of_range(const char*) __attribute__((__noreturn__));
void
__throw_out_of_range_fmt(const char*, ...) __attribute__((__noreturn__))
__attribute__((__format__(__gnu_printf__, 1, 2)));
void
__throw_runtime_error(const char*) __attribute__((__noreturn__));
void
__throw_range_error(const char*) __attribute__((__noreturn__));
void
__throw_overflow_error(const char*) __attribute__((__noreturn__));
void
__throw_underflow_error(const char*) __attribute__((__noreturn__));
void
__throw_ios_failure(const char*) __attribute__((__noreturn__));
void
__throw_ios_failure(const char*, int) __attribute__((__noreturn__));
void
__throw_system_error(int) __attribute__((__noreturn__));
void
__throw_future_error(int) __attribute__((__noreturn__));
void
__throw_bad_function_call() __attribute__((__noreturn__));
}
namespace __gnu_cxx __attribute__ ((__visibility__ ("default")))
{
template<typename _Tp>
struct __is_integer_nonstrict
: public std::__is_integer<_Tp>
{
using std::__is_integer<_Tp>::__value;
enum { __width = __value ? sizeof(_Tp) * 8 : 0 };
};
template<typename _Value>
struct __numeric_traits_integer
{
static_assert(__is_integer_nonstrict<_Value>::__value,
"invalid specialization");
static const bool __is_signed = (_Value)(-1) < 0;
static const int __digits
= __is_integer_nonstrict<_Value>::__width - __is_signed;
static const _Value __max = __is_signed
? (((((_Value)1 << (__digits - 1)) - 1) << 1) + 1)
: ~(_Value)0;
static const _Value __min = __is_signed ? -__max - 1 : (_Value)0;
};
template<typename _Value>
const _Value __numeric_traits_integer<_Value>::__min;
template<typename _Value>
const _Value __numeric_traits_integer<_Value>::__max;
template<typename _Value>
const bool __numeric_traits_integer<_Value>::__is_signed;
template<typename _Value>
const int __numeric_traits_integer<_Value>::__digits;
template<typename _Tp>
using __int_traits = __numeric_traits_integer<_Tp>;
template<typename _Value>
struct __numeric_traits_floating
{
static const int __max_digits10 = (2 + (std::__are_same<_Value, float>::__value ? 24 : std::__are_same<_Value, double>::__value ? 53 : 64) * 643L / 2136);
static const bool __is_signed = true;
static const int __digits10 = (std::__are_same<_Value, float>::__value ? 6 : std::__are_same<_Value, double>::__value ? 15 : 18);
static const int __max_exponent10 = (std::__are_same<_Value, float>::__value ? 38 : std::__are_same<_Value, double>::__value ? 308 : 4932);
};
template<typename _Value>
const int __numeric_traits_floating<_Value>::__max_digits10;
template<typename _Value>
const bool __numeric_traits_floating<_Value>::__is_signed;
template<typename _Value>
const int __numeric_traits_floating<_Value>::__digits10;
template<typename _Value>
const int __numeric_traits_floating<_Value>::__max_exponent10;
template<typename _Value>
struct __numeric_traits
: public __numeric_traits_integer<_Value>
{ };
template<>
struct __numeric_traits<float>
: public __numeric_traits_floating<float>
{ };
template<>
struct __numeric_traits<double>
: public __numeric_traits_floating<double>
{ };
template<>
struct __numeric_traits<long double>
: public __numeric_traits_floating<long double>
{ };
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Tp>
class reference_wrapper;
template<typename _Tp, _Tp __v>
struct integral_constant
{
static constexpr _Tp value = __v;
typedef _Tp value_type;
typedef integral_constant<_Tp, __v> type;
constexpr operator value_type() const noexcept { return value; }
constexpr value_type operator()() const noexcept { return value; }
};
using true_type = integral_constant<bool, true>;
using false_type = integral_constant<bool, false>;
template<bool __v>
using __bool_constant = integral_constant<bool, __v>;
template<bool __v>
using bool_constant = integral_constant<bool, __v>;
template<bool, typename _Tp = void>
struct enable_if
{ };
template<typename _Tp>
struct enable_if<true, _Tp>
{ typedef _Tp type; };
template<bool _Cond, typename _Tp = void>
using __enable_if_t = typename enable_if<_Cond, _Tp>::type;
template<bool>
struct __conditional
{
template<typename _Tp, typename>
using type = _Tp;
};
template<>
struct __conditional<false>
{
template<typename, typename _Up>
using type = _Up;
};
template<bool _Cond, typename _If, typename _Else>
using __conditional_t
= typename __conditional<_Cond>::template type<_If, _Else>;
template <typename _Type>
struct __type_identity
{ using type = _Type; };
template<typename _Tp>
using __type_identity_t = typename __type_identity<_Tp>::type;
namespace __detail
{
template<typename _Tp, typename...>
using __first_t = _Tp;
template<typename... _Bn>
auto __or_fn(int) -> __first_t<false_type,
__enable_if_t<!bool(_Bn::value)>...>;
template<typename... _Bn>
auto __or_fn(...) -> true_type;
template<typename... _Bn>
auto __and_fn(int) -> __first_t<true_type,
__enable_if_t<bool(_Bn::value)>...>;
template<typename... _Bn>
auto __and_fn(...) -> false_type;
}
template<typename... _Bn>
struct __or_
: decltype(__detail::__or_fn<_Bn...>(0))
{ };
template<typename... _Bn>
struct __and_
: decltype(__detail::__and_fn<_Bn...>(0))
{ };
template<typename _Pp>
struct __not_
: __bool_constant<!bool(_Pp::value)>
{ };
template<typename... _Bn>
inline constexpr bool __or_v = __or_<_Bn...>::value;
template<typename... _Bn>
inline constexpr bool __and_v = __and_<_Bn...>::value;
namespace __detail
{
template<typename , typename _B1, typename... _Bn>
struct __disjunction_impl
{ using type = _B1; };
template<typename _B1, typename _B2, typename... _Bn>
struct __disjunction_impl<__enable_if_t<!bool(_B1::value)>, _B1, _B2, _Bn...>
{ using type = typename __disjunction_impl<void, _B2, _Bn...>::type; };
template<typename , typename _B1, typename... _Bn>
struct __conjunction_impl
{ using type = _B1; };
template<typename _B1, typename _B2, typename... _Bn>
struct __conjunction_impl<__enable_if_t<bool(_B1::value)>, _B1, _B2, _Bn...>
{ using type = typename __conjunction_impl<void, _B2, _Bn...>::type; };
}
template<typename... _Bn>
struct conjunction
: __detail::__conjunction_impl<void, _Bn...>::type
{ };
template<>
struct conjunction<>
: true_type
{ };
template<typename... _Bn>
struct disjunction
: __detail::__disjunction_impl<void, _Bn...>::type
{ };
template<>
struct disjunction<>
: false_type
{ };
template<typename _Pp>
struct negation
: __not_<_Pp>::type
{ };
template<typename... _Bn>
inline constexpr bool conjunction_v = conjunction<_Bn...>::value;
template<typename... _Bn>
inline constexpr bool disjunction_v = disjunction<_Bn...>::value;
template<typename _Pp>
inline constexpr bool negation_v = negation<_Pp>::value;
template<typename>
struct is_reference;
template<typename>
struct is_function;
template<typename>
struct is_void;
template<typename>
struct remove_cv;
template<typename>
struct is_const;
template<typename>
struct __is_array_unknown_bounds;
template <typename _Tp, size_t = sizeof(_Tp)>
constexpr true_type __is_complete_or_unbounded(__type_identity<_Tp>)
{ return {}; }
template <typename _TypeIdentity,
typename _NestedType = typename _TypeIdentity::type>
constexpr typename __or_<
is_reference<_NestedType>,
is_function<_NestedType>,
is_void<_NestedType>,
__is_array_unknown_bounds<_NestedType>
>::type __is_complete_or_unbounded(_TypeIdentity)
{ return {}; }
template<typename _Tp>
using __remove_cv_t = typename remove_cv<_Tp>::type;
template<typename _Tp>
struct is_void
: public false_type { };
template<>
struct is_void<void>
: public true_type { };
template<>
struct is_void<const void>
: public true_type { };
template<>
struct is_void<volatile void>
: public true_type { };
template<>
struct is_void<const volatile void>
: public true_type { };
template<typename>
struct __is_integral_helper
: public false_type { };
template<>
struct __is_integral_helper<bool>
: public true_type { };
template<>
struct __is_integral_helper<char>
: public true_type { };
template<>
struct __is_integral_helper<signed char>
: public true_type { };
template<>
struct __is_integral_helper<unsigned char>
: public true_type { };
template<>
struct __is_integral_helper<wchar_t>
: public true_type { };
template<>
struct __is_integral_helper<char16_t>
: public true_type { };
template<>
struct __is_integral_helper<char32_t>
: public true_type { };
template<>
struct __is_integral_helper<short>
: public true_type { };
template<>
struct __is_integral_helper<unsigned short>
: public true_type { };
template<>
struct __is_integral_helper<int>
: public true_type { };
template<>
struct __is_integral_helper<unsigned int>
: public true_type { };
template<>
struct __is_integral_helper<long>
: public true_type { };
template<>
struct __is_integral_helper<unsigned long>
: public true_type { };
template<>
struct __is_integral_helper<long long>
: public true_type { };
template<>
struct __is_integral_helper<unsigned long long>
: public true_type { };
template<typename _Tp>
struct is_integral
: public __is_integral_helper<__remove_cv_t<_Tp>>::type
{ };
template<typename>
struct __is_floating_point_helper
: public false_type { };
template<>
struct __is_floating_point_helper<float>
: public true_type { };
template<>
struct __is_floating_point_helper<double>
: public true_type { };
template<>
struct __is_floating_point_helper<long double>
: public true_type { };
template<>
struct __is_floating_point_helper<__float128>
: public true_type { };
template<typename _Tp>
struct is_floating_point
: public __is_floating_point_helper<__remove_cv_t<_Tp>>::type
{ };
template<typename>
struct is_array
: public false_type { };
template<typename _Tp, std::size_t _Size>
struct is_array<_Tp[_Size]>
: public true_type { };
template<typename _Tp>
struct is_array<_Tp[]>
: public true_type { };
template<typename>
struct __is_pointer_helper
: public false_type { };
template<typename _Tp>
struct __is_pointer_helper<_Tp*>
: public true_type { };
template<typename _Tp>
struct is_pointer
: public __is_pointer_helper<__remove_cv_t<_Tp>>::type
{ };
template<typename>
struct is_lvalue_reference
: public false_type { };
template<typename _Tp>
struct is_lvalue_reference<_Tp&>
: public true_type { };
template<typename>
struct is_rvalue_reference
: public false_type { };
template<typename _Tp>
struct is_rvalue_reference<_Tp&&>
: public true_type { };
template<typename>
struct __is_member_object_pointer_helper
: public false_type { };
template<typename _Tp, typename _Cp>
struct __is_member_object_pointer_helper<_Tp _Cp::*>
: public __not_<is_function<_Tp>>::type { };
template<typename _Tp>
struct is_member_object_pointer
: public __is_member_object_pointer_helper<__remove_cv_t<_Tp>>::type
{ };
template<typename>
struct __is_member_function_pointer_helper
: public false_type { };
template<typename _Tp, typename _Cp>
struct __is_member_function_pointer_helper<_Tp _Cp::*>
: public is_function<_Tp>::type { };
template<typename _Tp>
struct is_member_function_pointer
: public __is_member_function_pointer_helper<__remove_cv_t<_Tp>>::type
{ };
template<typename _Tp>
struct is_enum
: public integral_constant<bool, __is_enum(_Tp)>
{ };
template<typename _Tp>
struct is_union
: public integral_constant<bool, __is_union(_Tp)>
{ };
template<typename _Tp>
struct is_class
: public integral_constant<bool, __is_class(_Tp)>
{ };
template<typename _Tp>
struct is_function
: public __bool_constant<!is_const<const _Tp>::value> { };
template<typename _Tp>
struct is_function<_Tp&>
: public false_type { };
template<typename _Tp>
struct is_function<_Tp&&>
: public false_type { };
template<typename _Tp>
struct is_null_pointer
: public false_type { };
template<>
struct is_null_pointer<std::nullptr_t>
: public true_type { };
template<>
struct is_null_pointer<const std::nullptr_t>
: public true_type { };
template<>
struct is_null_pointer<volatile std::nullptr_t>
: public true_type { };
template<>
struct is_null_pointer<const volatile std::nullptr_t>
: public true_type { };
template<typename _Tp>
struct __is_nullptr_t
: public is_null_pointer<_Tp>
{ } __attribute__ ((__deprecated__ ("use '" "std::is_null_pointer" "' instead")));
template<typename _Tp>
struct is_reference
: public false_type
{ };
template<typename _Tp>
struct is_reference<_Tp&>
: public true_type
{ };
template<typename _Tp>
struct is_reference<_Tp&&>
: public true_type
{ };
template<typename _Tp>
struct is_arithmetic
: public __or_<is_integral<_Tp>, is_floating_point<_Tp>>::type
{ };
template<typename _Tp>
struct is_fundamental
: public __or_<is_arithmetic<_Tp>, is_void<_Tp>,
is_null_pointer<_Tp>>::type
{ };
template<typename _Tp>
struct is_object
: public __not_<__or_<is_function<_Tp>, is_reference<_Tp>,
is_void<_Tp>>>::type
{ };
template<typename>
struct is_member_pointer;
template<typename _Tp>
struct is_scalar
: public __or_<is_arithmetic<_Tp>, is_enum<_Tp>, is_pointer<_Tp>,
is_member_pointer<_Tp>, is_null_pointer<_Tp>>::type
{ };
template<typename _Tp>
struct is_compound
: public __not_<is_fundamental<_Tp>>::type { };
template<typename _Tp>
struct __is_member_pointer_helper
: public false_type { };
template<typename _Tp, typename _Cp>
struct __is_member_pointer_helper<_Tp _Cp::*>
: public true_type { };
template<typename _Tp>
struct is_member_pointer
: public __is_member_pointer_helper<__remove_cv_t<_Tp>>::type
{ };
template<typename, typename>
struct is_same;
template<typename _Tp, typename... _Types>
using __is_one_of = __or_<is_same<_Tp, _Types>...>;
__extension__
template<typename _Tp>
using __is_signed_integer = __is_one_of<__remove_cv_t<_Tp>,
signed char, signed short, signed int, signed long,
signed long long
>;
__extension__
template<typename _Tp>
using __is_unsigned_integer = __is_one_of<__remove_cv_t<_Tp>,
unsigned char, unsigned short, unsigned int, unsigned long,
unsigned long long
>;
template<typename _Tp>
using __is_standard_integer
= __or_<__is_signed_integer<_Tp>, __is_unsigned_integer<_Tp>>;
template<typename...> using __void_t = void;
template<typename>
struct is_const
: public false_type { };
template<typename _Tp>
struct is_const<_Tp const>
: public true_type { };
template<typename>
struct is_volatile
: public false_type { };
template<typename _Tp>
struct is_volatile<_Tp volatile>
: public true_type { };
template<typename _Tp>
struct is_trivial
: public integral_constant<bool, __is_trivial(_Tp)>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct is_trivially_copyable
: public integral_constant<bool, __is_trivially_copyable(_Tp)>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct is_standard_layout
: public integral_constant<bool, __is_standard_layout(_Tp)>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct
is_pod
: public integral_constant<bool, __is_pod(_Tp)>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct
[[__deprecated__]]
is_literal_type
: public integral_constant<bool, __is_literal_type(_Tp)>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct is_empty
: public integral_constant<bool, __is_empty(_Tp)>
{ };
template<typename _Tp>
struct is_polymorphic
: public integral_constant<bool, __is_polymorphic(_Tp)>
{ };
template<typename _Tp>
struct is_final
: public integral_constant<bool, __is_final(_Tp)>
{ };
template<typename _Tp>
struct is_abstract
: public integral_constant<bool, __is_abstract(_Tp)>
{ };
template<typename _Tp,
bool = is_arithmetic<_Tp>::value>
struct __is_signed_helper
: public false_type { };
template<typename _Tp>
struct __is_signed_helper<_Tp, true>
: public integral_constant<bool, _Tp(-1) < _Tp(0)>
{ };
template<typename _Tp>
struct is_signed
: public __is_signed_helper<_Tp>::type
{ };
template<typename _Tp>
struct is_unsigned
: public __and_<is_arithmetic<_Tp>, __not_<is_signed<_Tp>>>::type
{ };
template<typename _Tp, typename _Up = _Tp&&>
_Up
__declval(int);
template<typename _Tp>
_Tp
__declval(long);
template<typename _Tp>
auto declval() noexcept -> decltype(__declval<_Tp>(0));
template<typename>
struct remove_all_extents;
template<typename _Tp>
struct __is_array_known_bounds
: public false_type
{ };
template<typename _Tp, size_t _Size>
struct __is_array_known_bounds<_Tp[_Size]>
: public true_type
{ };
template<typename _Tp>
struct __is_array_unknown_bounds
: public false_type
{ };
template<typename _Tp>
struct __is_array_unknown_bounds<_Tp[]>
: public true_type
{ };
struct __do_is_destructible_impl
{
template<typename _Tp, typename = decltype(declval<_Tp&>().~_Tp())>
static true_type __test(int);
template<typename>
static false_type __test(...);
};
template<typename _Tp>
struct __is_destructible_impl
: public __do_is_destructible_impl
{
typedef decltype(__test<_Tp>(0)) type;
};
template<typename _Tp,
bool = __or_<is_void<_Tp>,
__is_array_unknown_bounds<_Tp>,
is_function<_Tp>>::value,
bool = __or_<is_reference<_Tp>, is_scalar<_Tp>>::value>
struct __is_destructible_safe;
template<typename _Tp>
struct __is_destructible_safe<_Tp, false, false>
: public __is_destructible_impl<typename
remove_all_extents<_Tp>::type>::type
{ };
template<typename _Tp>
struct __is_destructible_safe<_Tp, true, false>
: public false_type { };
template<typename _Tp>
struct __is_destructible_safe<_Tp, false, true>
: public true_type { };
template<typename _Tp>
struct is_destructible
: public __is_destructible_safe<_Tp>::type
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
struct __do_is_nt_destructible_impl
{
template<typename _Tp>
static __bool_constant<noexcept(declval<_Tp&>().~_Tp())>
__test(int);
template<typename>
static false_type __test(...);
};
template<typename _Tp>
struct __is_nt_destructible_impl
: public __do_is_nt_destructible_impl
{
typedef decltype(__test<_Tp>(0)) type;
};
template<typename _Tp,
bool = __or_<is_void<_Tp>,
__is_array_unknown_bounds<_Tp>,
is_function<_Tp>>::value,
bool = __or_<is_reference<_Tp>, is_scalar<_Tp>>::value>
struct __is_nt_destructible_safe;
template<typename _Tp>
struct __is_nt_destructible_safe<_Tp, false, false>
: public __is_nt_destructible_impl<typename
remove_all_extents<_Tp>::type>::type
{ };
template<typename _Tp>
struct __is_nt_destructible_safe<_Tp, true, false>
: public false_type { };
template<typename _Tp>
struct __is_nt_destructible_safe<_Tp, false, true>
: public true_type { };
template<typename _Tp>
struct is_nothrow_destructible
: public __is_nt_destructible_safe<_Tp>::type
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp, typename... _Args>
using __is_constructible_impl
= __bool_constant<__is_constructible(_Tp, _Args...)>;
template<typename _Tp, typename... _Args>
struct is_constructible
: public __is_constructible_impl<_Tp, _Args...>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct is_default_constructible
: public __is_constructible_impl<_Tp>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp, typename = void>
struct __add_lvalue_reference_helper
{ using type = _Tp; };
template<typename _Tp>
struct __add_lvalue_reference_helper<_Tp, __void_t<_Tp&>>
{ using type = _Tp&; };
template<typename _Tp>
using __add_lval_ref_t = typename __add_lvalue_reference_helper<_Tp>::type;
template<typename _Tp>
struct is_copy_constructible
: public __is_constructible_impl<_Tp, __add_lval_ref_t<const _Tp>>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp, typename = void>
struct __add_rvalue_reference_helper
{ using type = _Tp; };
template<typename _Tp>
struct __add_rvalue_reference_helper<_Tp, __void_t<_Tp&&>>
{ using type = _Tp&&; };
template<typename _Tp>
using __add_rval_ref_t = typename __add_rvalue_reference_helper<_Tp>::type;
template<typename _Tp>
struct is_move_constructible
: public __is_constructible_impl<_Tp, __add_rval_ref_t<_Tp>>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp, typename... _Args>
using __is_nothrow_constructible_impl
= __bool_constant<__is_nothrow_constructible(_Tp, _Args...)>;
template<typename _Tp, typename... _Args>
struct is_nothrow_constructible
: public __is_nothrow_constructible_impl<_Tp, _Args...>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct is_nothrow_default_constructible
: public __is_nothrow_constructible_impl<_Tp>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct is_nothrow_copy_constructible
: public __is_nothrow_constructible_impl<_Tp, __add_lval_ref_t<const _Tp>>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct is_nothrow_move_constructible
: public __is_nothrow_constructible_impl<_Tp, __add_rval_ref_t<_Tp>>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp, typename _Up>
using __is_assignable_impl = __bool_constant<__is_assignable(_Tp, _Up)>;
template<typename _Tp, typename _Up>
struct is_assignable
: public __is_assignable_impl<_Tp, _Up>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct is_copy_assignable
: public __is_assignable_impl<__add_lval_ref_t<_Tp>,
__add_lval_ref_t<const _Tp>>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct is_move_assignable
: public __is_assignable_impl<__add_lval_ref_t<_Tp>, __add_rval_ref_t<_Tp>>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp, typename _Up>
using __is_nothrow_assignable_impl
= __bool_constant<__is_nothrow_assignable(_Tp, _Up)>;
template<typename _Tp, typename _Up>
struct is_nothrow_assignable
: public __is_nothrow_assignable_impl<_Tp, _Up>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct is_nothrow_copy_assignable
: public __is_nothrow_assignable_impl<__add_lval_ref_t<_Tp>,
__add_lval_ref_t<const _Tp>>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct is_nothrow_move_assignable
: public __is_nothrow_assignable_impl<__add_lval_ref_t<_Tp>,
__add_rval_ref_t<_Tp>>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp, typename... _Args>
using __is_trivially_constructible_impl
= __bool_constant<__is_trivially_constructible(_Tp, _Args...)>;
template<typename _Tp, typename... _Args>
struct is_trivially_constructible
: public __is_trivially_constructible_impl<_Tp, _Args...>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct is_trivially_default_constructible
: public __is_trivially_constructible_impl<_Tp>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
struct __do_is_implicitly_default_constructible_impl
{
template <typename _Tp>
static void __helper(const _Tp&);
template <typename _Tp>
static true_type __test(const _Tp&,
decltype(__helper<const _Tp&>({}))* = 0);
static false_type __test(...);
};
template<typename _Tp>
struct __is_implicitly_default_constructible_impl
: public __do_is_implicitly_default_constructible_impl
{
typedef decltype(__test(declval<_Tp>())) type;
};
template<typename _Tp>
struct __is_implicitly_default_constructible_safe
: public __is_implicitly_default_constructible_impl<_Tp>::type
{ };
template <typename _Tp>
struct __is_implicitly_default_constructible
: public __and_<__is_constructible_impl<_Tp>,
__is_implicitly_default_constructible_safe<_Tp>>::type
{ };
template<typename _Tp>
struct is_trivially_copy_constructible
: public __is_trivially_constructible_impl<_Tp, __add_lval_ref_t<const _Tp>>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct is_trivially_move_constructible
: public __is_trivially_constructible_impl<_Tp, __add_rval_ref_t<_Tp>>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp, typename _Up>
using __is_trivially_assignable_impl
= __bool_constant<__is_trivially_assignable(_Tp, _Up)>;
template<typename _Tp, typename _Up>
struct is_trivially_assignable
: public __is_trivially_assignable_impl<_Tp, _Up>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct is_trivially_copy_assignable
: public __is_trivially_assignable_impl<__add_lval_ref_t<_Tp>,
__add_lval_ref_t<const _Tp>>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct is_trivially_move_assignable
: public __is_trivially_assignable_impl<__add_lval_ref_t<_Tp>,
__add_rval_ref_t<_Tp>>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct is_trivially_destructible
: public __and_<__is_destructible_safe<_Tp>,
__bool_constant<__has_trivial_destructor(_Tp)>>::type
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct has_virtual_destructor
: public integral_constant<bool, __has_virtual_destructor(_Tp)>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct alignment_of
: public integral_constant<std::size_t, alignof(_Tp)>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename>
struct rank
: public integral_constant<std::size_t, 0> { };
template<typename _Tp, std::size_t _Size>
struct rank<_Tp[_Size]>
: public integral_constant<std::size_t, 1 + rank<_Tp>::value> { };
template<typename _Tp>
struct rank<_Tp[]>
: public integral_constant<std::size_t, 1 + rank<_Tp>::value> { };
template<typename, unsigned _Uint = 0>
struct extent
: public integral_constant<size_t, 0> { };
template<typename _Tp, size_t _Size>
struct extent<_Tp[_Size], 0>
: public integral_constant<size_t, _Size> { };
template<typename _Tp, unsigned _Uint, size_t _Size>
struct extent<_Tp[_Size], _Uint>
: public extent<_Tp, _Uint - 1>::type { };
template<typename _Tp>
struct extent<_Tp[], 0>
: public integral_constant<size_t, 0> { };
template<typename _Tp, unsigned _Uint>
struct extent<_Tp[], _Uint>
: public extent<_Tp, _Uint - 1>::type { };
template<typename _Tp, typename _Up>
struct is_same
: public integral_constant<bool, __is_same(_Tp, _Up)>
{ };
template<typename _Base, typename _Derived>
struct is_base_of
: public integral_constant<bool, __is_base_of(_Base, _Derived)>
{ };
template<typename _From, typename _To>
struct is_convertible
: public __bool_constant<__is_convertible(_From, _To)>
{ };
template<typename _ToElementType, typename _FromElementType>
using __is_array_convertible
= is_convertible<_FromElementType(*)[], _ToElementType(*)[]>;
template<typename _Tp>
struct remove_const
{ typedef _Tp type; };
template<typename _Tp>
struct remove_const<_Tp const>
{ typedef _Tp type; };
template<typename _Tp>
struct remove_volatile
{ typedef _Tp type; };
template<typename _Tp>
struct remove_volatile<_Tp volatile>
{ typedef _Tp type; };
template<typename _Tp>
struct remove_cv
{ using type = __remove_cv(_Tp); };
template<typename _Tp>
struct add_const
{ using type = _Tp const; };
template<typename _Tp>
struct add_volatile
{ using type = _Tp volatile; };
template<typename _Tp>
struct add_cv
{ using type = _Tp const volatile; };
template<typename _Tp>
using remove_const_t = typename remove_const<_Tp>::type;
template<typename _Tp>
using remove_volatile_t = typename remove_volatile<_Tp>::type;
template<typename _Tp>
using remove_cv_t = typename remove_cv<_Tp>::type;
template<typename _Tp>
using add_const_t = typename add_const<_Tp>::type;
template<typename _Tp>
using add_volatile_t = typename add_volatile<_Tp>::type;
template<typename _Tp>
using add_cv_t = typename add_cv<_Tp>::type;
template<typename _Tp>
struct remove_reference
{ using type = __remove_reference(_Tp); };
template<typename _Tp>
struct add_lvalue_reference
{ using type = __add_lval_ref_t<_Tp>; };
template<typename _Tp>
struct add_rvalue_reference
{ using type = __add_rval_ref_t<_Tp>; };
template<typename _Tp>
using remove_reference_t = typename remove_reference<_Tp>::type;
template<typename _Tp>
using add_lvalue_reference_t = typename add_lvalue_reference<_Tp>::type;
template<typename _Tp>
using add_rvalue_reference_t = typename add_rvalue_reference<_Tp>::type;
template<typename _Unqualified, bool _IsConst, bool _IsVol>
struct __cv_selector;
template<typename _Unqualified>
struct __cv_selector<_Unqualified, false, false>
{ typedef _Unqualified __type; };
template<typename _Unqualified>
struct __cv_selector<_Unqualified, false, true>
{ typedef volatile _Unqualified __type; };
template<typename _Unqualified>
struct __cv_selector<_Unqualified, true, false>
{ typedef const _Unqualified __type; };
template<typename _Unqualified>
struct __cv_selector<_Unqualified, true, true>
{ typedef const volatile _Unqualified __type; };
template<typename _Qualified, typename _Unqualified,
bool _IsConst = is_const<_Qualified>::value,
bool _IsVol = is_volatile<_Qualified>::value>
class __match_cv_qualifiers
{
typedef __cv_selector<_Unqualified, _IsConst, _IsVol> __match;
public:
typedef typename __match::__type __type;
};
template<typename _Tp>
struct __make_unsigned
{ typedef _Tp __type; };
template<>
struct __make_unsigned<char>
{ typedef unsigned char __type; };
template<>
struct __make_unsigned<signed char>
{ typedef unsigned char __type; };
template<>
struct __make_unsigned<short>
{ typedef unsigned short __type; };
template<>
struct __make_unsigned<int>
{ typedef unsigned int __type; };
template<>
struct __make_unsigned<long>
{ typedef unsigned long __type; };
template<>
struct __make_unsigned<long long>
{ typedef unsigned long long __type; };
template<typename _Tp,
bool _IsInt = is_integral<_Tp>::value,
bool _IsEnum = is_enum<_Tp>::value>
class __make_unsigned_selector;
template<typename _Tp>
class __make_unsigned_selector<_Tp, true, false>
{
using __unsigned_type
= typename __make_unsigned<__remove_cv_t<_Tp>>::__type;
public:
using __type
= typename __match_cv_qualifiers<_Tp, __unsigned_type>::__type;
};
class __make_unsigned_selector_base
{
protected:
template<typename...> struct _List { };
template<typename _Tp, typename... _Up>
struct _List<_Tp, _Up...> : _List<_Up...>
{ static constexpr size_t __size = sizeof(_Tp); };
template<size_t _Sz, typename _Tp, bool = (_Sz <= _Tp::__size)>
struct __select;
template<size_t _Sz, typename _Uint, typename... _UInts>
struct __select<_Sz, _List<_Uint, _UInts...>, true>
{ using __type = _Uint; };
template<size_t _Sz, typename _Uint, typename... _UInts>
struct __select<_Sz, _List<_Uint, _UInts...>, false>
: __select<_Sz, _List<_UInts...>>
{ };
};
template<typename _Tp>
class __make_unsigned_selector<_Tp, false, true>
: __make_unsigned_selector_base
{
using _UInts = _List<unsigned char, unsigned short, unsigned int,
unsigned long, unsigned long long>;
using __unsigned_type = typename __select<sizeof(_Tp), _UInts>::__type;
public:
using __type
= typename __match_cv_qualifiers<_Tp, __unsigned_type>::__type;
};
template<>
struct __make_unsigned<wchar_t>
{
using __type
= typename __make_unsigned_selector<wchar_t, false, true>::__type;
};
template<>
struct __make_unsigned<char16_t>
{
using __type
= typename __make_unsigned_selector<char16_t, false, true>::__type;
};
template<>
struct __make_unsigned<char32_t>
{
using __type
= typename __make_unsigned_selector<char32_t, false, true>::__type;
};
template<typename _Tp>
struct make_unsigned
{ typedef typename __make_unsigned_selector<_Tp>::__type type; };
template<> struct make_unsigned<bool>;
template<> struct make_unsigned<bool const>;
template<> struct make_unsigned<bool volatile>;
template<> struct make_unsigned<bool const volatile>;
template<typename _Tp>
struct __make_signed
{ typedef _Tp __type; };
template<>
struct __make_signed<char>
{ typedef signed char __type; };
template<>
struct __make_signed<unsigned char>
{ typedef signed char __type; };
template<>
struct __make_signed<unsigned short>
{ typedef signed short __type; };
template<>
struct __make_signed<unsigned int>
{ typedef signed int __type; };
template<>
struct __make_signed<unsigned long>
{ typedef signed long __type; };
template<>
struct __make_signed<unsigned long long>
{ typedef signed long long __type; };
template<typename _Tp,
bool _IsInt = is_integral<_Tp>::value,
bool _IsEnum = is_enum<_Tp>::value>
class __make_signed_selector;
template<typename _Tp>
class __make_signed_selector<_Tp, true, false>
{
using __signed_type
= typename __make_signed<__remove_cv_t<_Tp>>::__type;
public:
using __type
= typename __match_cv_qualifiers<_Tp, __signed_type>::__type;
};
template<typename _Tp>
class __make_signed_selector<_Tp, false, true>
{
typedef typename __make_unsigned_selector<_Tp>::__type __unsigned_type;
public:
typedef typename __make_signed_selector<__unsigned_type>::__type __type;
};
template<>
struct __make_signed<wchar_t>
{
using __type
= typename __make_signed_selector<wchar_t, false, true>::__type;
};
template<>
struct __make_signed<char16_t>
{
using __type
= typename __make_signed_selector<char16_t, false, true>::__type;
};
template<>
struct __make_signed<char32_t>
{
using __type
= typename __make_signed_selector<char32_t, false, true>::__type;
};
template<typename _Tp>
struct make_signed
{ typedef typename __make_signed_selector<_Tp>::__type type; };
template<> struct make_signed<bool>;
template<> struct make_signed<bool const>;
template<> struct make_signed<bool volatile>;
template<> struct make_signed<bool const volatile>;
template<typename _Tp>
using make_signed_t = typename make_signed<_Tp>::type;
template<typename _Tp>
using make_unsigned_t = typename make_unsigned<_Tp>::type;
template<typename _Tp>
struct remove_extent
{ typedef _Tp type; };
template<typename _Tp, std::size_t _Size>
struct remove_extent<_Tp[_Size]>
{ typedef _Tp type; };
template<typename _Tp>
struct remove_extent<_Tp[]>
{ typedef _Tp type; };
template<typename _Tp>
struct remove_all_extents
{ typedef _Tp type; };
template<typename _Tp, std::size_t _Size>
struct remove_all_extents<_Tp[_Size]>
{ typedef typename remove_all_extents<_Tp>::type type; };
template<typename _Tp>
struct remove_all_extents<_Tp[]>
{ typedef typename remove_all_extents<_Tp>::type type; };
template<typename _Tp>
using remove_extent_t = typename remove_extent<_Tp>::type;
template<typename _Tp>
using remove_all_extents_t = typename remove_all_extents<_Tp>::type;
template<typename _Tp, typename>
struct __remove_pointer_helper
{ typedef _Tp type; };
template<typename _Tp, typename _Up>
struct __remove_pointer_helper<_Tp, _Up*>
{ typedef _Up type; };
template<typename _Tp>
struct remove_pointer
: public __remove_pointer_helper<_Tp, __remove_cv_t<_Tp>>
{ };
template<typename _Tp, typename = void>
struct __add_pointer_helper
{ using type = _Tp; };
template<typename _Tp>
struct __add_pointer_helper<_Tp, __void_t<_Tp*>>
{ using type = _Tp*; };
template<typename _Tp>
struct add_pointer
: public __add_pointer_helper<_Tp>
{ };
template<typename _Tp>
struct add_pointer<_Tp&>
{ using type = _Tp*; };
template<typename _Tp>
struct add_pointer<_Tp&&>
{ using type = _Tp*; };
template<typename _Tp>
using remove_pointer_t = typename remove_pointer<_Tp>::type;
template<typename _Tp>
using add_pointer_t = typename add_pointer<_Tp>::type;
template<std::size_t _Len>
struct __aligned_storage_msa
{
union __type
{
unsigned char __data[_Len];
struct __attribute__((__aligned__)) { } __align;
};
};
template<std::size_t _Len, std::size_t _Align =
__alignof__(typename __aligned_storage_msa<_Len>::__type)>
struct
aligned_storage
{
union type
{
unsigned char __data[_Len];
struct __attribute__((__aligned__((_Align)))) { } __align;
};
};
template <typename... _Types>
struct __strictest_alignment
{
static const size_t _S_alignment = 0;
static const size_t _S_size = 0;
};
template <typename _Tp, typename... _Types>
struct __strictest_alignment<_Tp, _Types...>
{
static const size_t _S_alignment =
alignof(_Tp) > __strictest_alignment<_Types...>::_S_alignment
? alignof(_Tp) : __strictest_alignment<_Types...>::_S_alignment;
static const size_t _S_size =
sizeof(_Tp) > __strictest_alignment<_Types...>::_S_size
? sizeof(_Tp) : __strictest_alignment<_Types...>::_S_size;
};
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template <size_t _Len, typename... _Types>
struct
aligned_union
{
private:
static_assert(sizeof...(_Types) != 0, "At least one type is required");
using __strictest = __strictest_alignment<_Types...>;
static const size_t _S_len = _Len > __strictest::_S_size
? _Len : __strictest::_S_size;
public:
static const size_t alignment_value = __strictest::_S_alignment;
typedef typename aligned_storage<_S_len, alignment_value>::type type;
};
template <size_t _Len, typename... _Types>
const size_t aligned_union<_Len, _Types...>::alignment_value;
#pragma GCC diagnostic pop
template<typename _Up>
struct __decay_selector
: __conditional_t<is_const<const _Up>::value,
remove_cv<_Up>,
add_pointer<_Up>>
{ };
template<typename _Up, size_t _Nm>
struct __decay_selector<_Up[_Nm]>
{ using type = _Up*; };
template<typename _Up>
struct __decay_selector<_Up[]>
{ using type = _Up*; };
template<typename _Tp>
struct decay
{ using type = typename __decay_selector<_Tp>::type; };
template<typename _Tp>
struct decay<_Tp&>
{ using type = typename __decay_selector<_Tp>::type; };
template<typename _Tp>
struct decay<_Tp&&>
{ using type = typename __decay_selector<_Tp>::type; };
template<typename _Tp>
struct __strip_reference_wrapper
{
typedef _Tp __type;
};
template<typename _Tp>
struct __strip_reference_wrapper<reference_wrapper<_Tp> >
{
typedef _Tp& __type;
};
template<typename _Tp>
using __decay_t = typename decay<_Tp>::type;
template<typename _Tp>
using __decay_and_strip = __strip_reference_wrapper<__decay_t<_Tp>>;
template<typename... _Cond>
using _Require = __enable_if_t<__and_<_Cond...>::value>;
template<typename _Tp>
using __remove_cvref_t
= typename remove_cv<typename remove_reference<_Tp>::type>::type;
template<bool _Cond, typename _Iftrue, typename _Iffalse>
struct conditional
{ typedef _Iftrue type; };
template<typename _Iftrue, typename _Iffalse>
struct conditional<false, _Iftrue, _Iffalse>
{ typedef _Iffalse type; };
template<typename... _Tp>
struct common_type;
template<typename _Tp>
struct __success_type
{ typedef _Tp type; };
struct __failure_type
{ };
struct __do_common_type_impl
{
template<typename _Tp, typename _Up>
using __cond_t
= decltype(true ? std::declval<_Tp>() : std::declval<_Up>());
template<typename _Tp, typename _Up>
static __success_type<__decay_t<__cond_t<_Tp, _Up>>>
_S_test(int);
template<typename, typename>
static __failure_type
_S_test_2(...);
template<typename _Tp, typename _Up>
static decltype(_S_test_2<_Tp, _Up>(0))
_S_test(...);
};
template<>
struct common_type<>
{ };
template<typename _Tp0>
struct common_type<_Tp0>
: public common_type<_Tp0, _Tp0>
{ };
template<typename _Tp1, typename _Tp2,
typename _Dp1 = __decay_t<_Tp1>, typename _Dp2 = __decay_t<_Tp2>>
struct __common_type_impl
{
using type = common_type<_Dp1, _Dp2>;
};
template<typename _Tp1, typename _Tp2>
struct __common_type_impl<_Tp1, _Tp2, _Tp1, _Tp2>
: private __do_common_type_impl
{
using type = decltype(_S_test<_Tp1, _Tp2>(0));
};
template<typename _Tp1, typename _Tp2>
struct common_type<_Tp1, _Tp2>
: public __common_type_impl<_Tp1, _Tp2>::type
{ };
template<typename...>
struct __common_type_pack
{ };
template<typename, typename, typename = void>
struct __common_type_fold;
template<typename _Tp1, typename _Tp2, typename... _Rp>
struct common_type<_Tp1, _Tp2, _Rp...>
: public __common_type_fold<common_type<_Tp1, _Tp2>,
__common_type_pack<_Rp...>>
{ };
template<typename _CTp, typename... _Rp>
struct __common_type_fold<_CTp, __common_type_pack<_Rp...>,
__void_t<typename _CTp::type>>
: public common_type<typename _CTp::type, _Rp...>
{ };
template<typename _CTp, typename _Rp>
struct __common_type_fold<_CTp, _Rp, void>
{ };
template<typename _Tp, bool = is_enum<_Tp>::value>
struct __underlying_type_impl
{
using type = __underlying_type(_Tp);
};
template<typename _Tp>
struct __underlying_type_impl<_Tp, false>
{ };
template<typename _Tp>
struct underlying_type
: public __underlying_type_impl<_Tp>
{ };
template<typename _Tp>
struct __declval_protector
{
static const bool __stop = false;
};
template<typename _Tp>
auto declval() noexcept -> decltype(__declval<_Tp>(0))
{
static_assert(__declval_protector<_Tp>::__stop,
"declval() must not be used!");
return __declval<_Tp>(0);
}
template<typename _Signature>
struct result_of;
struct __invoke_memfun_ref { };
struct __invoke_memfun_deref { };
struct __invoke_memobj_ref { };
struct __invoke_memobj_deref { };
struct __invoke_other { };
template<typename _Tp, typename _Tag>
struct __result_of_success : __success_type<_Tp>
{ using __invoke_type = _Tag; };
struct __result_of_memfun_ref_impl
{
template<typename _Fp, typename _Tp1, typename... _Args>
static __result_of_success<decltype(
(std::declval<_Tp1>().*std::declval<_Fp>())(std::declval<_Args>()...)
), __invoke_memfun_ref> _S_test(int);
template<typename...>
static __failure_type _S_test(...);
};
template<typename _MemPtr, typename _Arg, typename... _Args>
struct __result_of_memfun_ref
: private __result_of_memfun_ref_impl
{
typedef decltype(_S_test<_MemPtr, _Arg, _Args...>(0)) type;
};
struct __result_of_memfun_deref_impl
{
template<typename _Fp, typename _Tp1, typename... _Args>
static __result_of_success<decltype(
((*std::declval<_Tp1>()).*std::declval<_Fp>())(std::declval<_Args>()...)
), __invoke_memfun_deref> _S_test(int);
template<typename...>
static __failure_type _S_test(...);
};
template<typename _MemPtr, typename _Arg, typename... _Args>
struct __result_of_memfun_deref
: private __result_of_memfun_deref_impl
{
typedef decltype(_S_test<_MemPtr, _Arg, _Args...>(0)) type;
};
struct __result_of_memobj_ref_impl
{
template<typename _Fp, typename _Tp1>
static __result_of_success<decltype(
std::declval<_Tp1>().*std::declval<_Fp>()
), __invoke_memobj_ref> _S_test(int);
template<typename, typename>
static __failure_type _S_test(...);
};
template<typename _MemPtr, typename _Arg>
struct __result_of_memobj_ref
: private __result_of_memobj_ref_impl
{
typedef decltype(_S_test<_MemPtr, _Arg>(0)) type;
};
struct __result_of_memobj_deref_impl
{
template<typename _Fp, typename _Tp1>
static __result_of_success<decltype(
(*std::declval<_Tp1>()).*std::declval<_Fp>()
), __invoke_memobj_deref> _S_test(int);
template<typename, typename>
static __failure_type _S_test(...);
};
template<typename _MemPtr, typename _Arg>
struct __result_of_memobj_deref
: private __result_of_memobj_deref_impl
{
typedef decltype(_S_test<_MemPtr, _Arg>(0)) type;
};
template<typename _MemPtr, typename _Arg>
struct __result_of_memobj;
template<typename _Res, typename _Class, typename _Arg>
struct __result_of_memobj<_Res _Class::*, _Arg>
{
typedef __remove_cvref_t<_Arg> _Argval;
typedef _Res _Class::* _MemPtr;
typedef typename __conditional_t<__or_<is_same<_Argval, _Class>,
is_base_of<_Class, _Argval>>::value,
__result_of_memobj_ref<_MemPtr, _Arg>,
__result_of_memobj_deref<_MemPtr, _Arg>
>::type type;
};
template<typename _MemPtr, typename _Arg, typename... _Args>
struct __result_of_memfun;
template<typename _Res, typename _Class, typename _Arg, typename... _Args>
struct __result_of_memfun<_Res _Class::*, _Arg, _Args...>
{
typedef typename remove_reference<_Arg>::type _Argval;
typedef _Res _Class::* _MemPtr;
typedef typename __conditional_t<is_base_of<_Class, _Argval>::value,
__result_of_memfun_ref<_MemPtr, _Arg, _Args...>,
__result_of_memfun_deref<_MemPtr, _Arg, _Args...>
>::type type;
};
template<typename _Tp, typename _Up = __remove_cvref_t<_Tp>>
struct __inv_unwrap
{
using type = _Tp;
};
template<typename _Tp, typename _Up>
struct __inv_unwrap<_Tp, reference_wrapper<_Up>>
{
using type = _Up&;
};
template<bool, bool, typename _Functor, typename... _ArgTypes>
struct __result_of_impl
{
typedef __failure_type type;
};
template<typename _MemPtr, typename _Arg>
struct __result_of_impl<true, false, _MemPtr, _Arg>
: public __result_of_memobj<__decay_t<_MemPtr>,
typename __inv_unwrap<_Arg>::type>
{ };
template<typename _MemPtr, typename _Arg, typename... _Args>
struct __result_of_impl<false, true, _MemPtr, _Arg, _Args...>
: public __result_of_memfun<__decay_t<_MemPtr>,
typename __inv_unwrap<_Arg>::type, _Args...>
{ };
struct __result_of_other_impl
{
template<typename _Fn, typename... _Args>
static __result_of_success<decltype(
std::declval<_Fn>()(std::declval<_Args>()...)
), __invoke_other> _S_test(int);
template<typename...>
static __failure_type _S_test(...);
};
template<typename _Functor, typename... _ArgTypes>
struct __result_of_impl<false, false, _Functor, _ArgTypes...>
: private __result_of_other_impl
{
typedef decltype(_S_test<_Functor, _ArgTypes...>(0)) type;
};
template<typename _Functor, typename... _ArgTypes>
struct __invoke_result
: public __result_of_impl<
is_member_object_pointer<
typename remove_reference<_Functor>::type
>::value,
is_member_function_pointer<
typename remove_reference<_Functor>::type
>::value,
_Functor, _ArgTypes...
>::type
{ };
template<typename _Functor, typename... _ArgTypes>
struct result_of<_Functor(_ArgTypes...)>
: public __invoke_result<_Functor, _ArgTypes...>
{ } __attribute__ ((__deprecated__ ("use '" "std::invoke_result" "' instead")));
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<size_t _Len, size_t _Align =
__alignof__(typename __aligned_storage_msa<_Len>::__type)>
using aligned_storage_t = typename aligned_storage<_Len, _Align>::type;
template <size_t _Len, typename... _Types>
using aligned_union_t = typename aligned_union<_Len, _Types...>::type;
#pragma GCC diagnostic pop
template<typename _Tp>
using decay_t = typename decay<_Tp>::type;
template<bool _Cond, typename _Tp = void>
using enable_if_t = typename enable_if<_Cond, _Tp>::type;
template<bool _Cond, typename _Iftrue, typename _Iffalse>
using conditional_t = typename conditional<_Cond, _Iftrue, _Iffalse>::type;
template<typename... _Tp>
using common_type_t = typename common_type<_Tp...>::type;
template<typename _Tp>
using underlying_type_t = typename underlying_type<_Tp>::type;
template<typename _Tp>
using result_of_t = typename result_of<_Tp>::type;
template<typename...> using void_t = void;
template<typename _Default, typename _AlwaysVoid,
template<typename...> class _Op, typename... _Args>
struct __detector
{
using type = _Default;
using __is_detected = false_type;
};
template<typename _Default, template<typename...> class _Op,
typename... _Args>
struct __detector<_Default, __void_t<_Op<_Args...>>, _Op, _Args...>
{
using type = _Op<_Args...>;
using __is_detected = true_type;
};
template<typename _Default, template<typename...> class _Op,
typename... _Args>
using __detected_or = __detector<_Default, void, _Op, _Args...>;
template<typename _Default, template<typename...> class _Op,
typename... _Args>
using __detected_or_t
= typename __detected_or<_Default, _Op, _Args...>::type;
template <typename _Tp>
struct __is_swappable;
template <typename _Tp>
struct __is_nothrow_swappable;
template<typename>
struct __is_tuple_like_impl : false_type
{ };
template<typename _Tp>
struct __is_tuple_like
: public __is_tuple_like_impl<__remove_cvref_t<_Tp>>::type
{ };
template<typename _Tp>
inline
_Require<__not_<__is_tuple_like<_Tp>>,
is_move_constructible<_Tp>,
is_move_assignable<_Tp>>
swap(_Tp&, _Tp&)
noexcept(__and_<is_nothrow_move_constructible<_Tp>,
is_nothrow_move_assignable<_Tp>>::value);
template<typename _Tp, size_t _Nm>
inline
__enable_if_t<__is_swappable<_Tp>::value>
swap(_Tp (&__a)[_Nm], _Tp (&__b)[_Nm])
noexcept(__is_nothrow_swappable<_Tp>::value);
namespace __swappable_details {
using std::swap;
struct __do_is_swappable_impl
{
template<typename _Tp, typename
= decltype(swap(std::declval<_Tp&>(), std::declval<_Tp&>()))>
static true_type __test(int);
template<typename>
static false_type __test(...);
};
struct __do_is_nothrow_swappable_impl
{
template<typename _Tp>
static __bool_constant<
noexcept(swap(std::declval<_Tp&>(), std::declval<_Tp&>()))
> __test(int);
template<typename>
static false_type __test(...);
};
}
template<typename _Tp>
struct __is_swappable_impl
: public __swappable_details::__do_is_swappable_impl
{
typedef decltype(__test<_Tp>(0)) type;
};
template<typename _Tp>
struct __is_nothrow_swappable_impl
: public __swappable_details::__do_is_nothrow_swappable_impl
{
typedef decltype(__test<_Tp>(0)) type;
};
template<typename _Tp>
struct __is_swappable
: public __is_swappable_impl<_Tp>::type
{ };
template<typename _Tp>
struct __is_nothrow_swappable
: public __is_nothrow_swappable_impl<_Tp>::type
{ };
template<typename _Tp>
struct is_swappable
: public __is_swappable_impl<_Tp>::type
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
struct is_nothrow_swappable
: public __is_nothrow_swappable_impl<_Tp>::type
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
inline constexpr bool is_swappable_v =
is_swappable<_Tp>::value;
template<typename _Tp>
inline constexpr bool is_nothrow_swappable_v =
is_nothrow_swappable<_Tp>::value;
namespace __swappable_with_details {
using std::swap;
struct __do_is_swappable_with_impl
{
template<typename _Tp, typename _Up, typename
= decltype(swap(std::declval<_Tp>(), std::declval<_Up>())),
typename
= decltype(swap(std::declval<_Up>(), std::declval<_Tp>()))>
static true_type __test(int);
template<typename, typename>
static false_type __test(...);
};
struct __do_is_nothrow_swappable_with_impl
{
template<typename _Tp, typename _Up>
static __bool_constant<
noexcept(swap(std::declval<_Tp>(), std::declval<_Up>()))
&&
noexcept(swap(std::declval<_Up>(), std::declval<_Tp>()))
> __test(int);
template<typename, typename>
static false_type __test(...);
};
}
template<typename _Tp, typename _Up>
struct __is_swappable_with_impl
: public __swappable_with_details::__do_is_swappable_with_impl
{
typedef decltype(__test<_Tp, _Up>(0)) type;
};
template<typename _Tp>
struct __is_swappable_with_impl<_Tp&, _Tp&>
: public __swappable_details::__do_is_swappable_impl
{
typedef decltype(__test<_Tp&>(0)) type;
};
template<typename _Tp, typename _Up>
struct __is_nothrow_swappable_with_impl
: public __swappable_with_details::__do_is_nothrow_swappable_with_impl
{
typedef decltype(__test<_Tp, _Up>(0)) type;
};
template<typename _Tp>
struct __is_nothrow_swappable_with_impl<_Tp&, _Tp&>
: public __swappable_details::__do_is_nothrow_swappable_impl
{
typedef decltype(__test<_Tp&>(0)) type;
};
template<typename _Tp, typename _Up>
struct is_swappable_with
: public __is_swappable_with_impl<_Tp, _Up>::type
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"first template argument must be a complete class or an unbounded array");
static_assert(std::__is_complete_or_unbounded(__type_identity<_Up>{}),
"second template argument must be a complete class or an unbounded array");
};
template<typename _Tp, typename _Up>
struct is_nothrow_swappable_with
: public __is_nothrow_swappable_with_impl<_Tp, _Up>::type
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"first template argument must be a complete class or an unbounded array");
static_assert(std::__is_complete_or_unbounded(__type_identity<_Up>{}),
"second template argument must be a complete class or an unbounded array");
};
template<typename _Tp, typename _Up>
inline constexpr bool is_swappable_with_v =
is_swappable_with<_Tp, _Up>::value;
template<typename _Tp, typename _Up>
inline constexpr bool is_nothrow_swappable_with_v =
is_nothrow_swappable_with<_Tp, _Up>::value;
template<typename _Result, typename _Ret,
bool = is_void<_Ret>::value, typename = void>
struct __is_invocable_impl
: false_type
{
using __nothrow_conv = false_type;
};
template<typename _Result, typename _Ret>
struct __is_invocable_impl<_Result, _Ret,
true,
__void_t<typename _Result::type>>
: true_type
{
using __nothrow_conv = true_type;
};
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wctor-dtor-privacy"
template<typename _Result, typename _Ret>
struct __is_invocable_impl<_Result, _Ret,
false,
__void_t<typename _Result::type>>
{
private:
using _Res_t = typename _Result::type;
static _Res_t _S_get() noexcept;
template<typename _Tp>
static void _S_conv(__type_identity_t<_Tp>) noexcept;
template<typename _Tp,
bool _Nothrow = noexcept(_S_conv<_Tp>(_S_get())),
typename = decltype(_S_conv<_Tp>(_S_get())),
bool _Dangle = __reference_converts_from_temporary(_Tp, _Res_t)
>
static __bool_constant<_Nothrow && !_Dangle>
_S_test(int);
template<typename _Tp, bool = false>
static false_type
_S_test(...);
public:
using type = decltype(_S_test<_Ret, true>(1));
using __nothrow_conv = decltype(_S_test<_Ret>(1));
};
#pragma GCC diagnostic pop
template<typename _Fn, typename... _ArgTypes>
struct __is_invocable
: __is_invocable_impl<__invoke_result<_Fn, _ArgTypes...>, void>::type
{ };
template<typename _Fn, typename _Tp, typename... _Args>
constexpr bool __call_is_nt(__invoke_memfun_ref)
{
using _Up = typename __inv_unwrap<_Tp>::type;
return noexcept((std::declval<_Up>().*std::declval<_Fn>())(
std::declval<_Args>()...));
}
template<typename _Fn, typename _Tp, typename... _Args>
constexpr bool __call_is_nt(__invoke_memfun_deref)
{
return noexcept(((*std::declval<_Tp>()).*std::declval<_Fn>())(
std::declval<_Args>()...));
}
template<typename _Fn, typename _Tp>
constexpr bool __call_is_nt(__invoke_memobj_ref)
{
using _Up = typename __inv_unwrap<_Tp>::type;
return noexcept(std::declval<_Up>().*std::declval<_Fn>());
}
template<typename _Fn, typename _Tp>
constexpr bool __call_is_nt(__invoke_memobj_deref)
{
return noexcept((*std::declval<_Tp>()).*std::declval<_Fn>());
}
template<typename _Fn, typename... _Args>
constexpr bool __call_is_nt(__invoke_other)
{
return noexcept(std::declval<_Fn>()(std::declval<_Args>()...));
}
template<typename _Result, typename _Fn, typename... _Args>
struct __call_is_nothrow
: __bool_constant<
std::__call_is_nt<_Fn, _Args...>(typename _Result::__invoke_type{})
>
{ };
template<typename _Fn, typename... _Args>
using __call_is_nothrow_
= __call_is_nothrow<__invoke_result<_Fn, _Args...>, _Fn, _Args...>;
template<typename _Fn, typename... _Args>
struct __is_nothrow_invocable
: __and_<__is_invocable<_Fn, _Args...>,
__call_is_nothrow_<_Fn, _Args...>>::type
{ };
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wctor-dtor-privacy"
struct __nonesuchbase {};
struct __nonesuch : private __nonesuchbase {
~__nonesuch() = delete;
__nonesuch(__nonesuch const&) = delete;
void operator=(__nonesuch const&) = delete;
};
#pragma GCC diagnostic pop
template<typename _Functor, typename... _ArgTypes>
struct invoke_result
: public __invoke_result<_Functor, _ArgTypes...>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Functor>{}),
"_Functor must be a complete class or an unbounded array");
static_assert((std::__is_complete_or_unbounded(
__type_identity<_ArgTypes>{}) && ...),
"each argument type must be a complete class or an unbounded array");
};
template<typename _Fn, typename... _Args>
using invoke_result_t = typename invoke_result<_Fn, _Args...>::type;
template<typename _Fn, typename... _ArgTypes>
struct is_invocable
: __is_invocable_impl<__invoke_result<_Fn, _ArgTypes...>, void>::type
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Fn>{}),
"_Fn must be a complete class or an unbounded array");
static_assert((std::__is_complete_or_unbounded(
__type_identity<_ArgTypes>{}) && ...),
"each argument type must be a complete class or an unbounded array");
};
template<typename _Ret, typename _Fn, typename... _ArgTypes>
struct is_invocable_r
: __is_invocable_impl<__invoke_result<_Fn, _ArgTypes...>, _Ret>::type
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Fn>{}),
"_Fn must be a complete class or an unbounded array");
static_assert((std::__is_complete_or_unbounded(
__type_identity<_ArgTypes>{}) && ...),
"each argument type must be a complete class or an unbounded array");
static_assert(std::__is_complete_or_unbounded(__type_identity<_Ret>{}),
"_Ret must be a complete class or an unbounded array");
};
template<typename _Fn, typename... _ArgTypes>
struct is_nothrow_invocable
: __and_<__is_invocable_impl<__invoke_result<_Fn, _ArgTypes...>, void>,
__call_is_nothrow_<_Fn, _ArgTypes...>>::type
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Fn>{}),
"_Fn must be a complete class or an unbounded array");
static_assert((std::__is_complete_or_unbounded(
__type_identity<_ArgTypes>{}) && ...),
"each argument type must be a complete class or an unbounded array");
};
template<typename _Result, typename _Ret>
using __is_nt_invocable_impl
= typename __is_invocable_impl<_Result, _Ret>::__nothrow_conv;
template<typename _Ret, typename _Fn, typename... _ArgTypes>
struct is_nothrow_invocable_r
: __and_<__is_nt_invocable_impl<__invoke_result<_Fn, _ArgTypes...>, _Ret>,
__call_is_nothrow_<_Fn, _ArgTypes...>>::type
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Fn>{}),
"_Fn must be a complete class or an unbounded array");
static_assert((std::__is_complete_or_unbounded(
__type_identity<_ArgTypes>{}) && ...),
"each argument type must be a complete class or an unbounded array");
static_assert(std::__is_complete_or_unbounded(__type_identity<_Ret>{}),
"_Ret must be a complete class or an unbounded array");
};
template <typename _Tp>
inline constexpr bool is_void_v = is_void<_Tp>::value;
template <typename _Tp>
inline constexpr bool is_null_pointer_v = is_null_pointer<_Tp>::value;
template <typename _Tp>
inline constexpr bool is_integral_v = is_integral<_Tp>::value;
template <typename _Tp>
inline constexpr bool is_floating_point_v = is_floating_point<_Tp>::value;
template <typename _Tp>
inline constexpr bool is_array_v = false;
template <typename _Tp>
inline constexpr bool is_array_v<_Tp[]> = true;
template <typename _Tp, size_t _Num>
inline constexpr bool is_array_v<_Tp[_Num]> = true;
template <typename _Tp>
inline constexpr bool is_pointer_v = is_pointer<_Tp>::value;
template <typename _Tp>
inline constexpr bool is_lvalue_reference_v = false;
template <typename _Tp>
inline constexpr bool is_lvalue_reference_v<_Tp&> = true;
template <typename _Tp>
inline constexpr bool is_rvalue_reference_v = false;
template <typename _Tp>
inline constexpr bool is_rvalue_reference_v<_Tp&&> = true;
template <typename _Tp>
inline constexpr bool is_member_object_pointer_v =
is_member_object_pointer<_Tp>::value;
template <typename _Tp>
inline constexpr bool is_member_function_pointer_v =
is_member_function_pointer<_Tp>::value;
template <typename _Tp>
inline constexpr bool is_enum_v = __is_enum(_Tp);
template <typename _Tp>
inline constexpr bool is_union_v = __is_union(_Tp);
template <typename _Tp>
inline constexpr bool is_class_v = __is_class(_Tp);
template <typename _Tp>
inline constexpr bool is_function_v = is_function<_Tp>::value;
template <typename _Tp>
inline constexpr bool is_reference_v = false;
template <typename _Tp>
inline constexpr bool is_reference_v<_Tp&> = true;
template <typename _Tp>
inline constexpr bool is_reference_v<_Tp&&> = true;
template <typename _Tp>
inline constexpr bool is_arithmetic_v = is_arithmetic<_Tp>::value;
template <typename _Tp>
inline constexpr bool is_fundamental_v = is_fundamental<_Tp>::value;
template <typename _Tp>
inline constexpr bool is_object_v = is_object<_Tp>::value;
template <typename _Tp>
inline constexpr bool is_scalar_v = is_scalar<_Tp>::value;
template <typename _Tp>
inline constexpr bool is_compound_v = is_compound<_Tp>::value;
template <typename _Tp>
inline constexpr bool is_member_pointer_v = is_member_pointer<_Tp>::value;
template <typename _Tp>
inline constexpr bool is_const_v = false;
template <typename _Tp>
inline constexpr bool is_const_v<const _Tp> = true;
template <typename _Tp>
inline constexpr bool is_volatile_v = false;
template <typename _Tp>
inline constexpr bool is_volatile_v<volatile _Tp> = true;
template <typename _Tp>
inline constexpr bool is_trivial_v = __is_trivial(_Tp);
template <typename _Tp>
inline constexpr bool is_trivially_copyable_v = __is_trivially_copyable(_Tp);
template <typename _Tp>
inline constexpr bool is_standard_layout_v = __is_standard_layout(_Tp);
template <typename _Tp>
inline constexpr bool is_pod_v = __is_pod(_Tp);
template <typename _Tp>
[[__deprecated__]]
inline constexpr bool is_literal_type_v = __is_literal_type(_Tp);
template <typename _Tp>
inline constexpr bool is_empty_v = __is_empty(_Tp);
template <typename _Tp>
inline constexpr bool is_polymorphic_v = __is_polymorphic(_Tp);
template <typename _Tp>
inline constexpr bool is_abstract_v = __is_abstract(_Tp);
template <typename _Tp>
inline constexpr bool is_final_v = __is_final(_Tp);
template <typename _Tp>
inline constexpr bool is_signed_v = is_signed<_Tp>::value;
template <typename _Tp>
inline constexpr bool is_unsigned_v = is_unsigned<_Tp>::value;
template <typename _Tp, typename... _Args>
inline constexpr bool is_constructible_v = __is_constructible(_Tp, _Args...);
template <typename _Tp>
inline constexpr bool is_default_constructible_v = __is_constructible(_Tp);
template <typename _Tp>
inline constexpr bool is_copy_constructible_v
= __is_constructible(_Tp, __add_lval_ref_t<const _Tp>);
template <typename _Tp>
inline constexpr bool is_move_constructible_v
= __is_constructible(_Tp, __add_rval_ref_t<_Tp>);
template <typename _Tp, typename _Up>
inline constexpr bool is_assignable_v = __is_assignable(_Tp, _Up);
template <typename _Tp>
inline constexpr bool is_copy_assignable_v
= __is_assignable(__add_lval_ref_t<_Tp>, __add_lval_ref_t<const _Tp>);
template <typename _Tp>
inline constexpr bool is_move_assignable_v
= __is_assignable(__add_lval_ref_t<_Tp>, __add_rval_ref_t<_Tp>);
template <typename _Tp>
inline constexpr bool is_destructible_v = is_destructible<_Tp>::value;
template <typename _Tp, typename... _Args>
inline constexpr bool is_trivially_constructible_v
= __is_trivially_constructible(_Tp, _Args...);
template <typename _Tp>
inline constexpr bool is_trivially_default_constructible_v
= __is_trivially_constructible(_Tp);
template <typename _Tp>
inline constexpr bool is_trivially_copy_constructible_v
= __is_trivially_constructible(_Tp, __add_lval_ref_t<const _Tp>);
template <typename _Tp>
inline constexpr bool is_trivially_move_constructible_v
= __is_trivially_constructible(_Tp, __add_rval_ref_t<_Tp>);
template <typename _Tp, typename _Up>
inline constexpr bool is_trivially_assignable_v
= __is_trivially_assignable(_Tp, _Up);
template <typename _Tp>
inline constexpr bool is_trivially_copy_assignable_v
= __is_trivially_assignable(__add_lval_ref_t<_Tp>,
__add_lval_ref_t<const _Tp>);
template <typename _Tp>
inline constexpr bool is_trivially_move_assignable_v
= __is_trivially_assignable(__add_lval_ref_t<_Tp>,
__add_rval_ref_t<_Tp>);
template <typename _Tp>
inline constexpr bool is_trivially_destructible_v =
is_trivially_destructible<_Tp>::value;
template <typename _Tp, typename... _Args>
inline constexpr bool is_nothrow_constructible_v
= __is_nothrow_constructible(_Tp, _Args...);
template <typename _Tp>
inline constexpr bool is_nothrow_default_constructible_v
= __is_nothrow_constructible(_Tp);
template <typename _Tp>
inline constexpr bool is_nothrow_copy_constructible_v
= __is_nothrow_constructible(_Tp, __add_lval_ref_t<const _Tp>);
template <typename _Tp>
inline constexpr bool is_nothrow_move_constructible_v
= __is_nothrow_constructible(_Tp, __add_rval_ref_t<_Tp>);
template <typename _Tp, typename _Up>
inline constexpr bool is_nothrow_assignable_v
= __is_nothrow_assignable(_Tp, _Up);
template <typename _Tp>
inline constexpr bool is_nothrow_copy_assignable_v
= __is_nothrow_assignable(__add_lval_ref_t<_Tp>,
__add_lval_ref_t<const _Tp>);
template <typename _Tp>
inline constexpr bool is_nothrow_move_assignable_v
= __is_nothrow_assignable(__add_lval_ref_t<_Tp>, __add_rval_ref_t<_Tp>);
template <typename _Tp>
inline constexpr bool is_nothrow_destructible_v =
is_nothrow_destructible<_Tp>::value;
template <typename _Tp>
inline constexpr bool has_virtual_destructor_v
= __has_virtual_destructor(_Tp);
template <typename _Tp>
inline constexpr size_t alignment_of_v = alignment_of<_Tp>::value;
template <typename _Tp>
inline constexpr size_t rank_v = 0;
template <typename _Tp, size_t _Size>
inline constexpr size_t rank_v<_Tp[_Size]> = 1 + rank_v<_Tp>;
template <typename _Tp>
inline constexpr size_t rank_v<_Tp[]> = 1 + rank_v<_Tp>;
template <typename _Tp, unsigned _Idx = 0>
inline constexpr size_t extent_v = 0;
template <typename _Tp, size_t _Size>
inline constexpr size_t extent_v<_Tp[_Size], 0> = _Size;
template <typename _Tp, unsigned _Idx, size_t _Size>
inline constexpr size_t extent_v<_Tp[_Size], _Idx> = extent_v<_Tp, _Idx - 1>;
template <typename _Tp>
inline constexpr size_t extent_v<_Tp[], 0> = 0;
template <typename _Tp, unsigned _Idx>
inline constexpr size_t extent_v<_Tp[], _Idx> = extent_v<_Tp, _Idx - 1>;
template <typename _Tp, typename _Up>
inline constexpr bool is_same_v = __is_same(_Tp, _Up);
template <typename _Base, typename _Derived>
inline constexpr bool is_base_of_v = __is_base_of(_Base, _Derived);
template <typename _From, typename _To>
inline constexpr bool is_convertible_v = __is_convertible(_From, _To);
template<typename _Fn, typename... _Args>
inline constexpr bool is_invocable_v = is_invocable<_Fn, _Args...>::value;
template<typename _Fn, typename... _Args>
inline constexpr bool is_nothrow_invocable_v
= is_nothrow_invocable<_Fn, _Args...>::value;
template<typename _Ret, typename _Fn, typename... _Args>
inline constexpr bool is_invocable_r_v
= is_invocable_r<_Ret, _Fn, _Args...>::value;
template<typename _Ret, typename _Fn, typename... _Args>
inline constexpr bool is_nothrow_invocable_r_v
= is_nothrow_invocable_r<_Ret, _Fn, _Args...>::value;
template<typename _Tp>
struct has_unique_object_representations
: bool_constant<__has_unique_object_representations(
remove_cv_t<remove_all_extents_t<_Tp>>
)>
{
static_assert(std::__is_complete_or_unbounded(__type_identity<_Tp>{}),
"template argument must be a complete class or an unbounded array");
};
template<typename _Tp>
inline constexpr bool has_unique_object_representations_v
= has_unique_object_representations<_Tp>::value;
template<typename _Tp>
struct is_aggregate
: bool_constant<__is_aggregate(remove_cv_t<_Tp>)>
{ };
template<typename _Tp>
inline constexpr bool is_aggregate_v = __is_aggregate(remove_cv_t<_Tp>);
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Tp>
inline constexpr _Tp*
__addressof(_Tp& __r) noexcept
{ return __builtin_addressof(__r); }
template<typename _Tp>
[[__nodiscard__]]
constexpr _Tp&&
forward(typename std::remove_reference<_Tp>::type& __t) noexcept
{ return static_cast<_Tp&&>(__t); }
template<typename _Tp>
[[__nodiscard__]]
constexpr _Tp&&
forward(typename std::remove_reference<_Tp>::type&& __t) noexcept
{
static_assert(!std::is_lvalue_reference<_Tp>::value,
"std::forward must not be used to convert an rvalue to an lvalue");
return static_cast<_Tp&&>(__t);
}
template<typename _Tp>
[[__nodiscard__]]
constexpr typename std::remove_reference<_Tp>::type&&
move(_Tp&& __t) noexcept
{ return static_cast<typename std::remove_reference<_Tp>::type&&>(__t); }
template<typename _Tp>
struct __move_if_noexcept_cond
: public __and_<__not_<is_nothrow_move_constructible<_Tp>>,
is_copy_constructible<_Tp>>::type { };
template<typename _Tp>
[[__nodiscard__]]
constexpr
__conditional_t<__move_if_noexcept_cond<_Tp>::value, const _Tp&, _Tp&&>
move_if_noexcept(_Tp& __x) noexcept
{ return std::move(__x); }
template<typename _Tp>
[[__nodiscard__]]
inline constexpr _Tp*
addressof(_Tp& __r) noexcept
{ return std::__addressof(__r); }
template<typename _Tp>
const _Tp* addressof(const _Tp&&) = delete;
template <typename _Tp, typename _Up = _Tp>
inline _Tp
__exchange(_Tp& __obj, _Up&& __new_val)
{
_Tp __old_val = std::move(__obj);
__obj = std::forward<_Up>(__new_val);
return __old_val;
}
template<typename _Tp>
inline
typename enable_if<__and_<__not_<__is_tuple_like<_Tp>>,
is_move_constructible<_Tp>,
is_move_assignable<_Tp>>::value>::type
swap(_Tp& __a, _Tp& __b)
noexcept(__and_<is_nothrow_move_constructible<_Tp>, is_nothrow_move_assignable<_Tp>>::value)
{
_Tp __tmp = std::move(__a);
__a = std::move(__b);
__b = std::move(__tmp);
}
template<typename _Tp, size_t _Nm>
inline
typename enable_if<__is_swappable<_Tp>::value>::type
swap(_Tp (&__a)[_Nm], _Tp (&__b)[_Nm])
noexcept(__is_nothrow_swappable<_Tp>::value)
{
for (size_t __n = 0; __n < _Nm; ++__n)
swap(__a[__n], __b[__n]);
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Tp>
struct tuple_size;
template<typename _Tp,
typename _Up = typename remove_cv<_Tp>::type,
typename = typename enable_if<is_same<_Tp, _Up>::value>::type,
size_t = tuple_size<_Tp>::value>
using __enable_if_has_tuple_size = _Tp;
template<typename _Tp>
struct tuple_size<const __enable_if_has_tuple_size<_Tp>>
: public tuple_size<_Tp> { };
template<typename _Tp>
struct tuple_size<volatile __enable_if_has_tuple_size<_Tp>>
: public tuple_size<_Tp> { };
template<typename _Tp>
struct tuple_size<const volatile __enable_if_has_tuple_size<_Tp>>
: public tuple_size<_Tp> { };
template<typename _Tp>
inline constexpr size_t tuple_size_v = tuple_size<_Tp>::value;
template<size_t __i, typename _Tp>
struct tuple_element;
template<size_t __i, typename _Tp>
using __tuple_element_t = typename tuple_element<__i, _Tp>::type;
template<size_t __i, typename _Tp>
struct tuple_element<__i, const _Tp>
{
using type = const __tuple_element_t<__i, _Tp>;
};
template<size_t __i, typename _Tp>
struct tuple_element<__i, volatile _Tp>
{
using type = volatile __tuple_element_t<__i, _Tp>;
};
template<size_t __i, typename _Tp>
struct tuple_element<__i, const volatile _Tp>
{
using type = const volatile __tuple_element_t<__i, _Tp>;
};
template<typename _Tp, typename... _Types>
constexpr size_t
__find_uniq_type_in_pack()
{
constexpr size_t __sz = sizeof...(_Types);
constexpr bool __found[__sz] = { __is_same(_Tp, _Types) ... };
size_t __n = __sz;
for (size_t __i = 0; __i < __sz; ++__i)
{
if (__found[__i])
{
if (__n < __sz)
return __sz;
__n = __i;
}
}
return __n;
}
template<size_t __i, typename _Tp>
using tuple_element_t = typename tuple_element<__i, _Tp>::type;
template<size_t... _Indexes> struct _Index_tuple { };
template<size_t _Num>
struct _Build_index_tuple
{
using __type = _Index_tuple<__integer_pack(_Num)...>;
};
template<typename _Tp, _Tp... _Idx>
struct integer_sequence
{
typedef _Tp value_type;
static constexpr size_t size() noexcept { return sizeof...(_Idx); }
};
template<typename _Tp, _Tp _Num>
using make_integer_sequence
= integer_sequence<_Tp, __integer_pack(_Num)...>;
template<size_t... _Idx>
using index_sequence = integer_sequence<size_t, _Idx...>;
template<size_t _Num>
using make_index_sequence = make_integer_sequence<size_t, _Num>;
template<typename... _Types>
using index_sequence_for = make_index_sequence<sizeof...(_Types)>;
struct in_place_t {
explicit in_place_t() = default;
};
inline constexpr in_place_t in_place{};
template<typename _Tp> struct in_place_type_t
{
explicit in_place_type_t() = default;
};
template<typename _Tp>
inline constexpr in_place_type_t<_Tp> in_place_type{};
template<size_t _Idx> struct in_place_index_t
{
explicit in_place_index_t() = default;
};
template<size_t _Idx>
inline constexpr in_place_index_t<_Idx> in_place_index{};
template<typename>
inline constexpr bool __is_in_place_type_v = false;
template<typename _Tp>
inline constexpr bool __is_in_place_type_v<in_place_type_t<_Tp>> = true;
template<typename _Tp>
using __is_in_place_type = bool_constant<__is_in_place_type_v<_Tp>>;
template<size_t _Np, typename... _Types>
struct _Nth_type
{ };
template<typename _Tp0, typename... _Rest>
struct _Nth_type<0, _Tp0, _Rest...>
{ using type = _Tp0; };
template<typename _Tp0, typename _Tp1, typename... _Rest>
struct _Nth_type<1, _Tp0, _Tp1, _Rest...>
{ using type = _Tp1; };
template<typename _Tp0, typename _Tp1, typename _Tp2, typename... _Rest>
struct _Nth_type<2, _Tp0, _Tp1, _Tp2, _Rest...>
{ using type = _Tp2; };
template<size_t _Np, typename _Tp0, typename _Tp1, typename _Tp2,
typename... _Rest>
struct _Nth_type<_Np, _Tp0, _Tp1, _Tp2, _Rest...>
: _Nth_type<_Np - 3, _Rest...>
{ };
template<typename _Tp0, typename _Tp1, typename... _Rest>
struct _Nth_type<0, _Tp0, _Tp1, _Rest...>
{ using type = _Tp0; };
template<typename _Tp0, typename _Tp1, typename _Tp2, typename... _Rest>
struct _Nth_type<0, _Tp0, _Tp1, _Tp2, _Rest...>
{ using type = _Tp0; };
template<typename _Tp0, typename _Tp1, typename _Tp2, typename... _Rest>
struct _Nth_type<1, _Tp0, _Tp1, _Tp2, _Rest...>
{ using type = _Tp1; };
}
namespace std __attribute__ ((__visibility__ ("default")))
{
struct piecewise_construct_t { explicit piecewise_construct_t() = default; };
inline constexpr piecewise_construct_t piecewise_construct =
piecewise_construct_t();
template<typename...>
class tuple;
template<size_t...>
struct _Index_tuple;
template <bool, typename _T1, typename _T2>
struct _PCC
{
template <typename _U1, typename _U2>
static constexpr bool _ConstructiblePair()
{
return __and_<is_constructible<_T1, const _U1&>,
is_constructible<_T2, const _U2&>>::value;
}
template <typename _U1, typename _U2>
static constexpr bool _ImplicitlyConvertiblePair()
{
return __and_<is_convertible<const _U1&, _T1>,
is_convertible<const _U2&, _T2>>::value;
}
template <typename _U1, typename _U2>
static constexpr bool _MoveConstructiblePair()
{
return __and_<is_constructible<_T1, _U1&&>,
is_constructible<_T2, _U2&&>>::value;
}
template <typename _U1, typename _U2>
static constexpr bool _ImplicitlyMoveConvertiblePair()
{
return __and_<is_convertible<_U1&&, _T1>,
is_convertible<_U2&&, _T2>>::value;
}
};
template <typename _T1, typename _T2>
struct _PCC<false, _T1, _T2>
{
template <typename _U1, typename _U2>
static constexpr bool _ConstructiblePair()
{
return false;
}
template <typename _U1, typename _U2>
static constexpr bool _ImplicitlyConvertiblePair()
{
return false;
}
template <typename _U1, typename _U2>
static constexpr bool _MoveConstructiblePair()
{
return false;
}
template <typename _U1, typename _U2>
static constexpr bool _ImplicitlyMoveConvertiblePair()
{
return false;
}
};
template<typename _U1, typename _U2> class __pair_base
{
template<typename _T1, typename _T2> friend struct pair;
__pair_base() = default;
~__pair_base() = default;
__pair_base(const __pair_base&) = default;
__pair_base& operator=(const __pair_base&) = delete;
};
template<typename _T1, typename _T2>
struct pair
: public __pair_base<_T1, _T2>
{
typedef _T1 first_type;
typedef _T2 second_type;
_T1 first;
_T2 second;
constexpr pair(const pair&) = default;
constexpr pair(pair&&) = default;
template<typename... _Args1, typename... _Args2>
pair(piecewise_construct_t, tuple<_Args1...>, tuple<_Args2...>);
void
swap(pair& __p)
noexcept(__and_<__is_nothrow_swappable<_T1>,
__is_nothrow_swappable<_T2>>::value)
{
using std::swap;
swap(first, __p.first);
swap(second, __p.second);
}
private:
template<typename... _Args1, size_t... _Indexes1,
typename... _Args2, size_t... _Indexes2>
pair(tuple<_Args1...>&, tuple<_Args2...>&,
_Index_tuple<_Indexes1...>, _Index_tuple<_Indexes2...>);
public:
template <typename _U1 = _T1,
typename _U2 = _T2,
typename enable_if<__and_<
__is_implicitly_default_constructible<_U1>,
__is_implicitly_default_constructible<_U2>>
::value, bool>::type = true>
constexpr pair()
: first(), second() { }
template <typename _U1 = _T1,
typename _U2 = _T2,
typename enable_if<__and_<
is_default_constructible<_U1>,
is_default_constructible<_U2>,
__not_<
__and_<__is_implicitly_default_constructible<_U1>,
__is_implicitly_default_constructible<_U2>>>>
::value, bool>::type = false>
explicit constexpr pair()
: first(), second() { }
using _PCCP = _PCC<true, _T1, _T2>;
template<typename _U1 = _T1, typename _U2=_T2, typename
enable_if<_PCCP::template
_ConstructiblePair<_U1, _U2>()
&& _PCCP::template
_ImplicitlyConvertiblePair<_U1, _U2>(),
bool>::type=true>
constexpr pair(const _T1& __a, const _T2& __b)
: first(__a), second(__b) { }
template<typename _U1 = _T1, typename _U2=_T2, typename
enable_if<_PCCP::template
_ConstructiblePair<_U1, _U2>()
&& !_PCCP::template
_ImplicitlyConvertiblePair<_U1, _U2>(),
bool>::type=false>
explicit constexpr pair(const _T1& __a, const _T2& __b)
: first(__a), second(__b) { }
template <typename _U1, typename _U2>
using _PCCFP = _PCC<!is_same<_T1, _U1>::value
|| !is_same<_T2, _U2>::value,
_T1, _T2>;
template<typename _U1, typename _U2, typename
enable_if<_PCCFP<_U1, _U2>::template
_ConstructiblePair<_U1, _U2>()
&& _PCCFP<_U1, _U2>::template
_ImplicitlyConvertiblePair<_U1, _U2>(),
bool>::type=true>
constexpr pair(const pair<_U1, _U2>& __p)
: first(__p.first), second(__p.second)
{ ; }
template<typename _U1, typename _U2, typename
enable_if<_PCCFP<_U1, _U2>::template
_ConstructiblePair<_U1, _U2>()
&& !_PCCFP<_U1, _U2>::template
_ImplicitlyConvertiblePair<_U1, _U2>(),
bool>::type=false>
explicit constexpr pair(const pair<_U1, _U2>& __p)
: first(__p.first), second(__p.second)
{ ; }
private:
struct __zero_as_null_pointer_constant
{
__zero_as_null_pointer_constant(int __zero_as_null_pointer_constant::*)
{ }
template<typename _Tp,
typename = __enable_if_t<is_null_pointer<_Tp>::value>>
__zero_as_null_pointer_constant(_Tp) = delete;
};
public:
template<typename _U1,
__enable_if_t<__and_<__not_<is_reference<_U1>>,
is_pointer<_T2>,
is_constructible<_T1, _U1>,
__not_<is_constructible<_T1, const _U1&>>,
is_convertible<_U1, _T1>>::value,
bool> = true>
__attribute__ ((__deprecated__ ("use 'nullptr' instead of '0' to " "initialize std::pair of move-only " "type and pointer")))
constexpr
pair(_U1&& __x, __zero_as_null_pointer_constant, ...)
: first(std::forward<_U1>(__x)), second(nullptr)
{ ; }
template<typename _U1,
__enable_if_t<__and_<__not_<is_reference<_U1>>,
is_pointer<_T2>,
is_constructible<_T1, _U1>,
__not_<is_constructible<_T1, const _U1&>>,
__not_<is_convertible<_U1, _T1>>>::value,
bool> = false>
__attribute__ ((__deprecated__ ("use 'nullptr' instead of '0' to " "initialize std::pair of move-only " "type and pointer")))
explicit constexpr
pair(_U1&& __x, __zero_as_null_pointer_constant, ...)
: first(std::forward<_U1>(__x)), second(nullptr)
{ ; }
template<typename _U2,
__enable_if_t<__and_<is_pointer<_T1>,
__not_<is_reference<_U2>>,
is_constructible<_T2, _U2>,
__not_<is_constructible<_T2, const _U2&>>,
is_convertible<_U2, _T2>>::value,
bool> = true>
__attribute__ ((__deprecated__ ("use 'nullptr' instead of '0' to " "initialize std::pair of move-only " "type and pointer")))
constexpr
pair(__zero_as_null_pointer_constant, _U2&& __y, ...)
: first(nullptr), second(std::forward<_U2>(__y))
{ ; }
template<typename _U2,
__enable_if_t<__and_<is_pointer<_T1>,
__not_<is_reference<_U2>>,
is_constructible<_T2, _U2>,
__not_<is_constructible<_T2, const _U2&>>,
__not_<is_convertible<_U2, _T2>>>::value,
bool> = false>
__attribute__ ((__deprecated__ ("use 'nullptr' instead of '0' to " "initialize std::pair of move-only " "type and pointer")))
explicit constexpr
pair(__zero_as_null_pointer_constant, _U2&& __y, ...)
: first(nullptr), second(std::forward<_U2>(__y))
{ ; }
template<typename _U1, typename _U2, typename
enable_if<_PCCP::template
_MoveConstructiblePair<_U1, _U2>()
&& _PCCP::template
_ImplicitlyMoveConvertiblePair<_U1, _U2>(),
bool>::type=true>
constexpr pair(_U1&& __x, _U2&& __y)
: first(std::forward<_U1>(__x)), second(std::forward<_U2>(__y))
{ ; }
template<typename _U1, typename _U2, typename
enable_if<_PCCP::template
_MoveConstructiblePair<_U1, _U2>()
&& !_PCCP::template
_ImplicitlyMoveConvertiblePair<_U1, _U2>(),
bool>::type=false>
explicit constexpr pair(_U1&& __x, _U2&& __y)
: first(std::forward<_U1>(__x)), second(std::forward<_U2>(__y))
{ ; }
template<typename _U1, typename _U2, typename
enable_if<_PCCFP<_U1, _U2>::template
_MoveConstructiblePair<_U1, _U2>()
&& _PCCFP<_U1, _U2>::template
_ImplicitlyMoveConvertiblePair<_U1, _U2>(),
bool>::type=true>
constexpr pair(pair<_U1, _U2>&& __p)
: first(std::forward<_U1>(__p.first)),
second(std::forward<_U2>(__p.second))
{ ; }
template<typename _U1, typename _U2, typename
enable_if<_PCCFP<_U1, _U2>::template
_MoveConstructiblePair<_U1, _U2>()
&& !_PCCFP<_U1, _U2>::template
_ImplicitlyMoveConvertiblePair<_U1, _U2>(),
bool>::type=false>
explicit constexpr pair(pair<_U1, _U2>&& __p)
: first(std::forward<_U1>(__p.first)),
second(std::forward<_U2>(__p.second))
{ ; }
pair&
operator=(__conditional_t<__and_<is_copy_assignable<_T1>,
is_copy_assignable<_T2>>::value,
const pair&, const __nonesuch&> __p)
{
first = __p.first;
second = __p.second;
return *this;
}
pair&
operator=(__conditional_t<__and_<is_move_assignable<_T1>,
is_move_assignable<_T2>>::value,
pair&&, __nonesuch&&> __p)
noexcept(__and_<is_nothrow_move_assignable<_T1>,
is_nothrow_move_assignable<_T2>>::value)
{
first = std::forward<first_type>(__p.first);
second = std::forward<second_type>(__p.second);
return *this;
}
template<typename _U1, typename _U2>
typename enable_if<__and_<is_assignable<_T1&, const _U1&>,
is_assignable<_T2&, const _U2&>>::value,
pair&>::type
operator=(const pair<_U1, _U2>& __p)
{
first = __p.first;
second = __p.second;
return *this;
}
template<typename _U1, typename _U2>
typename enable_if<__and_<is_assignable<_T1&, _U1&&>,
is_assignable<_T2&, _U2&&>>::value,
pair&>::type
operator=(pair<_U1, _U2>&& __p)
{
first = std::forward<_U1>(__p.first);
second = std::forward<_U2>(__p.second);
return *this;
}
};
template<typename _T1, typename _T2> pair(_T1, _T2) -> pair<_T1, _T2>;
template<typename _T1, typename _T2>
inline constexpr bool
operator==(const pair<_T1, _T2>& __x, const pair<_T1, _T2>& __y)
{ return __x.first == __y.first && __x.second == __y.second; }
template<typename _T1, typename _T2>
inline constexpr bool
operator<(const pair<_T1, _T2>& __x, const pair<_T1, _T2>& __y)
{ return __x.first < __y.first
|| (!(__y.first < __x.first) && __x.second < __y.second); }
template<typename _T1, typename _T2>
inline constexpr bool
operator!=(const pair<_T1, _T2>& __x, const pair<_T1, _T2>& __y)
{ return !(__x == __y); }
template<typename _T1, typename _T2>
inline constexpr bool
operator>(const pair<_T1, _T2>& __x, const pair<_T1, _T2>& __y)
{ return __y < __x; }
template<typename _T1, typename _T2>
inline constexpr bool
operator<=(const pair<_T1, _T2>& __x, const pair<_T1, _T2>& __y)
{ return !(__y < __x); }
template<typename _T1, typename _T2>
inline constexpr bool
operator>=(const pair<_T1, _T2>& __x, const pair<_T1, _T2>& __y)
{ return !(__x < __y); }
template<typename _T1, typename _T2>
inline
typename enable_if<__and_<__is_swappable<_T1>,
__is_swappable<_T2>>::value>::type
swap(pair<_T1, _T2>& __x, pair<_T1, _T2>& __y)
noexcept(noexcept(__x.swap(__y)))
{ __x.swap(__y); }
template<typename _T1, typename _T2>
typename enable_if<!__and_<__is_swappable<_T1>,
__is_swappable<_T2>>::value>::type
swap(pair<_T1, _T2>&, pair<_T1, _T2>&) = delete;
template<typename _T1, typename _T2>
constexpr pair<typename __decay_and_strip<_T1>::__type,
typename __decay_and_strip<_T2>::__type>
make_pair(_T1&& __x, _T2&& __y)
{
typedef typename __decay_and_strip<_T1>::__type __ds_type1;
typedef typename __decay_and_strip<_T2>::__type __ds_type2;
typedef pair<__ds_type1, __ds_type2> __pair_type;
return __pair_type(std::forward<_T1>(__x), std::forward<_T2>(__y));
}
template<typename _T1, typename _T2>
struct __is_tuple_like_impl<pair<_T1, _T2>> : true_type
{ };
template<class _Tp1, class _Tp2>
struct tuple_size<pair<_Tp1, _Tp2>>
: public integral_constant<size_t, 2> { };
template<class _Tp1, class _Tp2>
struct tuple_element<0, pair<_Tp1, _Tp2>>
{ typedef _Tp1 type; };
template<class _Tp1, class _Tp2>
struct tuple_element<1, pair<_Tp1, _Tp2>>
{ typedef _Tp2 type; };
template<typename _Tp1, typename _Tp2>
inline constexpr size_t tuple_size_v<pair<_Tp1, _Tp2>> = 2;
template<typename _Tp1, typename _Tp2>
inline constexpr size_t tuple_size_v<const pair<_Tp1, _Tp2>> = 2;
template<typename _Tp>
inline constexpr bool __is_pair = false;
template<typename _Tp, typename _Up>
inline constexpr bool __is_pair<pair<_Tp, _Up>> = true;
template<size_t _Int>
struct __pair_get;
template<>
struct __pair_get<0>
{
template<typename _Tp1, typename _Tp2>
static constexpr _Tp1&
__get(pair<_Tp1, _Tp2>& __pair) noexcept
{ return __pair.first; }
template<typename _Tp1, typename _Tp2>
static constexpr _Tp1&&
__move_get(pair<_Tp1, _Tp2>&& __pair) noexcept
{ return std::forward<_Tp1>(__pair.first); }
template<typename _Tp1, typename _Tp2>
static constexpr const _Tp1&
__const_get(const pair<_Tp1, _Tp2>& __pair) noexcept
{ return __pair.first; }
template<typename _Tp1, typename _Tp2>
static constexpr const _Tp1&&
__const_move_get(const pair<_Tp1, _Tp2>&& __pair) noexcept
{ return std::forward<const _Tp1>(__pair.first); }
};
template<>
struct __pair_get<1>
{
template<typename _Tp1, typename _Tp2>
static constexpr _Tp2&
__get(pair<_Tp1, _Tp2>& __pair) noexcept
{ return __pair.second; }
template<typename _Tp1, typename _Tp2>
static constexpr _Tp2&&
__move_get(pair<_Tp1, _Tp2>&& __pair) noexcept
{ return std::forward<_Tp2>(__pair.second); }
template<typename _Tp1, typename _Tp2>
static constexpr const _Tp2&
__const_get(const pair<_Tp1, _Tp2>& __pair) noexcept
{ return __pair.second; }
template<typename _Tp1, typename _Tp2>
static constexpr const _Tp2&&
__const_move_get(const pair<_Tp1, _Tp2>&& __pair) noexcept
{ return std::forward<const _Tp2>(__pair.second); }
};
template<size_t _Int, class _Tp1, class _Tp2>
constexpr typename tuple_element<_Int, pair<_Tp1, _Tp2>>::type&
get(pair<_Tp1, _Tp2>& __in) noexcept
{ return __pair_get<_Int>::__get(__in); }
template<size_t _Int, class _Tp1, class _Tp2>
constexpr typename tuple_element<_Int, pair<_Tp1, _Tp2>>::type&&
get(pair<_Tp1, _Tp2>&& __in) noexcept
{ return __pair_get<_Int>::__move_get(std::move(__in)); }
template<size_t _Int, class _Tp1, class _Tp2>
constexpr const typename tuple_element<_Int, pair<_Tp1, _Tp2>>::type&
get(const pair<_Tp1, _Tp2>& __in) noexcept
{ return __pair_get<_Int>::__const_get(__in); }
template<size_t _Int, class _Tp1, class _Tp2>
constexpr const typename tuple_element<_Int, pair<_Tp1, _Tp2>>::type&&
get(const pair<_Tp1, _Tp2>&& __in) noexcept
{ return __pair_get<_Int>::__const_move_get(std::move(__in)); }
template <typename _Tp, typename _Up>
constexpr _Tp&
get(pair<_Tp, _Up>& __p) noexcept
{ return __p.first; }
template <typename _Tp, typename _Up>
constexpr const _Tp&
get(const pair<_Tp, _Up>& __p) noexcept
{ return __p.first; }
template <typename _Tp, typename _Up>
constexpr _Tp&&
get(pair<_Tp, _Up>&& __p) noexcept
{ return std::move(__p.first); }
template <typename _Tp, typename _Up>
constexpr const _Tp&&
get(const pair<_Tp, _Up>&& __p) noexcept
{ return std::move(__p.first); }
template <typename _Tp, typename _Up>
constexpr _Tp&
get(pair<_Up, _Tp>& __p) noexcept
{ return __p.second; }
template <typename _Tp, typename _Up>
constexpr const _Tp&
get(const pair<_Up, _Tp>& __p) noexcept
{ return __p.second; }
template <typename _Tp, typename _Up>
constexpr _Tp&&
get(pair<_Up, _Tp>&& __p) noexcept
{ return std::move(__p.second); }
template <typename _Tp, typename _Up>
constexpr const _Tp&&
get(const pair<_Up, _Tp>&& __p) noexcept
{ return std::move(__p.second); }
}
namespace std __attribute__ ((__visibility__ ("default")))
{
struct input_iterator_tag { };
struct output_iterator_tag { };
struct forward_iterator_tag : public input_iterator_tag { };
struct bidirectional_iterator_tag : public forward_iterator_tag { };
struct random_access_iterator_tag : public bidirectional_iterator_tag { };
template<typename _Category, typename _Tp, typename _Distance = ptrdiff_t,
typename _Pointer = _Tp*, typename _Reference = _Tp&>
struct [[__deprecated__]] iterator
{
typedef _Category iterator_category;
typedef _Tp value_type;
typedef _Distance difference_type;
typedef _Pointer pointer;
typedef _Reference reference;
};
template<typename _Iterator>
struct iterator_traits;
template<typename _Iterator, typename = __void_t<>>
struct __iterator_traits { };
template<typename _Iterator>
struct __iterator_traits<_Iterator,
__void_t<typename _Iterator::iterator_category,
typename _Iterator::value_type,
typename _Iterator::difference_type,
typename _Iterator::pointer,
typename _Iterator::reference>>
{
typedef typename _Iterator::iterator_category iterator_category;
typedef typename _Iterator::value_type value_type;
typedef typename _Iterator::difference_type difference_type;
typedef typename _Iterator::pointer pointer;
typedef typename _Iterator::reference reference;
};
template<typename _Iterator>
struct iterator_traits
: public __iterator_traits<_Iterator> { };
template<typename _Tp>
struct iterator_traits<_Tp*>
{
typedef random_access_iterator_tag iterator_category;
typedef _Tp value_type;
typedef ptrdiff_t difference_type;
typedef _Tp* pointer;
typedef _Tp& reference;
};
template<typename _Tp>
struct iterator_traits<const _Tp*>
{
typedef random_access_iterator_tag iterator_category;
typedef _Tp value_type;
typedef ptrdiff_t difference_type;
typedef const _Tp* pointer;
typedef const _Tp& reference;
};
template<typename _Iter>
__attribute__((__always_inline__))
inline constexpr
typename iterator_traits<_Iter>::iterator_category
__iterator_category(const _Iter&)
{ return typename iterator_traits<_Iter>::iterator_category(); }
template<typename _Iter>
using __iterator_category_t
= typename iterator_traits<_Iter>::iterator_category;
template<typename _InIter>
using _RequireInputIter =
__enable_if_t<is_convertible<__iterator_category_t<_InIter>,
input_iterator_tag>::value>;
template<typename _It,
typename _Cat = __iterator_category_t<_It>>
struct __is_random_access_iter
: is_base_of<random_access_iterator_tag, _Cat>
{
typedef is_base_of<random_access_iterator_tag, _Cat> _Base;
enum { __value = _Base::value };
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template <typename> struct _List_iterator;
template <typename> struct _List_const_iterator;
template<typename _InputIterator>
inline constexpr
typename iterator_traits<_InputIterator>::difference_type
__distance(_InputIterator __first, _InputIterator __last,
input_iterator_tag)
{
typename iterator_traits<_InputIterator>::difference_type __n = 0;
while (__first != __last)
{
++__first;
++__n;
}
return __n;
}
template<typename _RandomAccessIterator>
__attribute__((__always_inline__))
inline constexpr
typename iterator_traits<_RandomAccessIterator>::difference_type
__distance(_RandomAccessIterator __first, _RandomAccessIterator __last,
random_access_iterator_tag)
{
return __last - __first;
}
template<typename _Tp>
ptrdiff_t
__distance(std::_List_iterator<_Tp>,
std::_List_iterator<_Tp>,
input_iterator_tag);
template<typename _Tp>
ptrdiff_t
__distance(std::_List_const_iterator<_Tp>,
std::_List_const_iterator<_Tp>,
input_iterator_tag);
template<typename _OutputIterator>
void
__distance(_OutputIterator, _OutputIterator, output_iterator_tag) = delete;
template<typename _InputIterator>
[[__nodiscard__]] __attribute__((__always_inline__))
inline constexpr
typename iterator_traits<_InputIterator>::difference_type
distance(_InputIterator __first, _InputIterator __last)
{
return std::__distance(__first, __last,
std::__iterator_category(__first));
}
template<typename _InputIterator, typename _Distance>
inline constexpr void
__advance(_InputIterator& __i, _Distance __n, input_iterator_tag)
{
do { if (std::__is_constant_evaluated() && !bool(__n >= 0)) __builtin_unreachable(); } while (false);
while (__n--)
++__i;
}
template<typename _BidirectionalIterator, typename _Distance>
inline constexpr void
__advance(_BidirectionalIterator& __i, _Distance __n,
bidirectional_iterator_tag)
{
if (__n > 0)
while (__n--)
++__i;
else
while (__n++)
--__i;
}
template<typename _RandomAccessIterator, typename _Distance>
inline constexpr void
__advance(_RandomAccessIterator& __i, _Distance __n,
random_access_iterator_tag)
{
if (__builtin_constant_p(__n) && __n == 1)
++__i;
else if (__builtin_constant_p(__n) && __n == -1)
--__i;
else
__i += __n;
}
template<typename _OutputIterator, typename _Distance>
void
__advance(_OutputIterator&, _Distance, output_iterator_tag) = delete;
template<typename _InputIterator, typename _Distance>
__attribute__((__always_inline__))
inline constexpr void
advance(_InputIterator& __i, _Distance __n)
{
typename iterator_traits<_InputIterator>::difference_type __d = __n;
std::__advance(__i, __d, std::__iterator_category(__i));
}
template<typename _InputIterator>
[[__nodiscard__]] [[__gnu__::__always_inline__]]
inline constexpr _InputIterator
next(_InputIterator __x, typename
iterator_traits<_InputIterator>::difference_type __n = 1)
{
std::advance(__x, __n);
return __x;
}
template<typename _BidirectionalIterator>
[[__nodiscard__]] [[__gnu__::__always_inline__]]
inline constexpr _BidirectionalIterator
prev(_BidirectionalIterator __x, typename
iterator_traits<_BidirectionalIterator>::difference_type __n = 1)
{
std::advance(__x, -__n);
return __x;
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
class __undefined;
template<typename _Tp>
struct __get_first_arg
{ using type = __undefined; };
template<template<typename, typename...> class _SomeTemplate, typename _Tp,
typename... _Types>
struct __get_first_arg<_SomeTemplate<_Tp, _Types...>>
{ using type = _Tp; };
template<typename _Tp, typename _Up>
struct __replace_first_arg
{ };
template<template<typename, typename...> class _SomeTemplate, typename _Up,
typename _Tp, typename... _Types>
struct __replace_first_arg<_SomeTemplate<_Tp, _Types...>, _Up>
{ using type = _SomeTemplate<_Up, _Types...>; };
template<typename _Ptr, typename = void>
struct __ptr_traits_elem : __get_first_arg<_Ptr>
{ };
template<typename _Ptr>
struct __ptr_traits_elem<_Ptr, __void_t<typename _Ptr::element_type>>
{ using type = typename _Ptr::element_type; };
template<typename _Ptr>
using __ptr_traits_elem_t = typename __ptr_traits_elem<_Ptr>::type;
template<typename _Ptr, typename _Elt, bool = is_void<_Elt>::value>
struct __ptr_traits_ptr_to
{
using pointer = _Ptr;
using element_type = _Elt;
static pointer
pointer_to(element_type& __r)
{ return pointer::pointer_to(__r); }
};
template<typename _Ptr, typename _Elt>
struct __ptr_traits_ptr_to<_Ptr, _Elt, true>
{ };
template<typename _Tp>
struct __ptr_traits_ptr_to<_Tp*, _Tp, false>
{
using pointer = _Tp*;
using element_type = _Tp;
static pointer
pointer_to(element_type& __r) noexcept
{ return std::addressof(__r); }
};
template<typename _Ptr, typename _Elt>
struct __ptr_traits_impl : __ptr_traits_ptr_to<_Ptr, _Elt>
{
private:
template<typename _Tp>
using __diff_t = typename _Tp::difference_type;
template<typename _Tp, typename _Up>
using __rebind = __type_identity<typename _Tp::template rebind<_Up>>;
public:
using pointer = _Ptr;
using element_type = _Elt;
using difference_type = __detected_or_t<ptrdiff_t, __diff_t, _Ptr>;
template<typename _Up>
using rebind = typename __detected_or_t<__replace_first_arg<_Ptr, _Up>,
__rebind, _Ptr, _Up>::type;
};
template<typename _Ptr>
struct __ptr_traits_impl<_Ptr, __undefined>
{ };
template<typename _Ptr>
struct pointer_traits : __ptr_traits_impl<_Ptr, __ptr_traits_elem_t<_Ptr>>
{ };
template<typename _Tp>
struct pointer_traits<_Tp*> : __ptr_traits_ptr_to<_Tp*, _Tp>
{
typedef _Tp* pointer;
typedef _Tp element_type;
typedef ptrdiff_t difference_type;
template<typename _Up> using rebind = _Up*;
};
template<typename _Ptr, typename _Tp>
using __ptr_rebind = typename pointer_traits<_Ptr>::template rebind<_Tp>;
template<typename _Tp>
constexpr _Tp*
__to_address(_Tp* __ptr) noexcept
{
static_assert(!std::is_function<_Tp>::value, "not a function pointer");
return __ptr;
}
template<typename _Ptr>
constexpr typename std::pointer_traits<_Ptr>::element_type*
__to_address(const _Ptr& __ptr)
{ return std::__to_address(__ptr.operator->()); }
}
namespace std __attribute__ ((__visibility__ ("default")))
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Iterator>
class reverse_iterator
: public iterator<typename iterator_traits<_Iterator>::iterator_category,
typename iterator_traits<_Iterator>::value_type,
typename iterator_traits<_Iterator>::difference_type,
typename iterator_traits<_Iterator>::pointer,
typename iterator_traits<_Iterator>::reference>
{
template<typename _Iter>
friend class reverse_iterator;
protected:
_Iterator current;
typedef iterator_traits<_Iterator> __traits_type;
public:
typedef _Iterator iterator_type;
typedef typename __traits_type::pointer pointer;
typedef typename __traits_type::difference_type difference_type;
typedef typename __traits_type::reference reference;
constexpr
reverse_iterator()
noexcept(noexcept(_Iterator()))
: current()
{ }
explicit constexpr
reverse_iterator(iterator_type __x)
noexcept(noexcept(_Iterator(__x)))
: current(__x)
{ }
constexpr
reverse_iterator(const reverse_iterator& __x)
noexcept(noexcept(_Iterator(__x.current)))
: current(__x.current)
{ }
reverse_iterator& operator=(const reverse_iterator&) = default;
template<typename _Iter>
constexpr
reverse_iterator(const reverse_iterator<_Iter>& __x)
noexcept(noexcept(_Iterator(__x.current)))
: current(__x.current)
{ }
template<typename _Iter>
constexpr
reverse_iterator&
operator=(const reverse_iterator<_Iter>& __x)
noexcept(noexcept(current = __x.current))
{
current = __x.current;
return *this;
}
[[__nodiscard__]]
constexpr iterator_type
base() const
noexcept(noexcept(_Iterator(current)))
{ return current; }
[[__nodiscard__]]
constexpr reference
operator*() const
{
_Iterator __tmp = current;
return *--__tmp;
}
[[__nodiscard__]]
constexpr pointer
operator->() const
{
_Iterator __tmp = current;
--__tmp;
return _S_to_pointer(__tmp);
}
constexpr reverse_iterator&
operator++()
{
--current;
return *this;
}
constexpr reverse_iterator
operator++(int)
{
reverse_iterator __tmp = *this;
--current;
return __tmp;
}
constexpr reverse_iterator&
operator--()
{
++current;
return *this;
}
constexpr reverse_iterator
operator--(int)
{
reverse_iterator __tmp = *this;
++current;
return __tmp;
}
[[__nodiscard__]]
constexpr reverse_iterator
operator+(difference_type __n) const
{ return reverse_iterator(current - __n); }
constexpr reverse_iterator&
operator+=(difference_type __n)
{
current -= __n;
return *this;
}
[[__nodiscard__]]
constexpr reverse_iterator
operator-(difference_type __n) const
{ return reverse_iterator(current + __n); }
constexpr reverse_iterator&
operator-=(difference_type __n)
{
current += __n;
return *this;
}
[[__nodiscard__]]
constexpr reference
operator[](difference_type __n) const
{ return *(*this + __n); }
private:
template<typename _Tp>
static constexpr _Tp*
_S_to_pointer(_Tp* __p)
{ return __p; }
template<typename _Tp>
static constexpr pointer
_S_to_pointer(_Tp __t)
{ return __t.operator->(); }
};
template<typename _Iterator>
[[__nodiscard__]]
inline constexpr bool
operator==(const reverse_iterator<_Iterator>& __x,
const reverse_iterator<_Iterator>& __y)
{ return __x.base() == __y.base(); }
template<typename _Iterator>
[[__nodiscard__]]
inline constexpr bool
operator<(const reverse_iterator<_Iterator>& __x,
const reverse_iterator<_Iterator>& __y)
{ return __y.base() < __x.base(); }
template<typename _Iterator>
[[__nodiscard__]]
inline constexpr bool
operator!=(const reverse_iterator<_Iterator>& __x,
const reverse_iterator<_Iterator>& __y)
{ return !(__x == __y); }
template<typename _Iterator>
[[__nodiscard__]]
inline constexpr bool
operator>(const reverse_iterator<_Iterator>& __x,
const reverse_iterator<_Iterator>& __y)
{ return __y < __x; }
template<typename _Iterator>
[[__nodiscard__]]
inline constexpr bool
operator<=(const reverse_iterator<_Iterator>& __x,
const reverse_iterator<_Iterator>& __y)
{ return !(__y < __x); }
template<typename _Iterator>
[[__nodiscard__]]
inline constexpr bool
operator>=(const reverse_iterator<_Iterator>& __x,
const reverse_iterator<_Iterator>& __y)
{ return !(__x < __y); }
template<typename _IteratorL, typename _IteratorR>
[[__nodiscard__]]
inline constexpr bool
operator==(const reverse_iterator<_IteratorL>& __x,
const reverse_iterator<_IteratorR>& __y)
{ return __x.base() == __y.base(); }
template<typename _IteratorL, typename _IteratorR>
[[__nodiscard__]]
inline constexpr bool
operator<(const reverse_iterator<_IteratorL>& __x,
const reverse_iterator<_IteratorR>& __y)
{ return __x.base() > __y.base(); }
template<typename _IteratorL, typename _IteratorR>
[[__nodiscard__]]
inline constexpr bool
operator!=(const reverse_iterator<_IteratorL>& __x,
const reverse_iterator<_IteratorR>& __y)
{ return __x.base() != __y.base(); }
template<typename _IteratorL, typename _IteratorR>
[[__nodiscard__]]
inline constexpr bool
operator>(const reverse_iterator<_IteratorL>& __x,
const reverse_iterator<_IteratorR>& __y)
{ return __x.base() < __y.base(); }
template<typename _IteratorL, typename _IteratorR>
inline constexpr bool
operator<=(const reverse_iterator<_IteratorL>& __x,
const reverse_iterator<_IteratorR>& __y)
{ return __x.base() >= __y.base(); }
template<typename _IteratorL, typename _IteratorR>
[[__nodiscard__]]
inline constexpr bool
operator>=(const reverse_iterator<_IteratorL>& __x,
const reverse_iterator<_IteratorR>& __y)
{ return __x.base() <= __y.base(); }
template<typename _IteratorL, typename _IteratorR>
[[__nodiscard__]]
inline constexpr auto
operator-(const reverse_iterator<_IteratorL>& __x,
const reverse_iterator<_IteratorR>& __y)
-> decltype(__y.base() - __x.base())
{ return __y.base() - __x.base(); }
template<typename _Iterator>
[[__nodiscard__]]
inline constexpr reverse_iterator<_Iterator>
operator+(typename reverse_iterator<_Iterator>::difference_type __n,
const reverse_iterator<_Iterator>& __x)
{ return reverse_iterator<_Iterator>(__x.base() - __n); }
template<typename _Iterator>
inline constexpr reverse_iterator<_Iterator>
__make_reverse_iterator(_Iterator __i)
{ return reverse_iterator<_Iterator>(__i); }
template<typename _Iterator>
[[__nodiscard__]]
inline constexpr reverse_iterator<_Iterator>
make_reverse_iterator(_Iterator __i)
{ return reverse_iterator<_Iterator>(__i); }
template<typename _Iterator>
auto
__niter_base(reverse_iterator<_Iterator> __it)
-> decltype(__make_reverse_iterator(__niter_base(__it.base())))
{ return __make_reverse_iterator(__niter_base(__it.base())); }
template<typename _Iterator>
struct __is_move_iterator<reverse_iterator<_Iterator> >
: __is_move_iterator<_Iterator>
{ };
template<typename _Iterator>
auto
__miter_base(reverse_iterator<_Iterator> __it)
-> decltype(__make_reverse_iterator(__miter_base(__it.base())))
{ return __make_reverse_iterator(__miter_base(__it.base())); }
template<typename _Container>
class back_insert_iterator
: public iterator<output_iterator_tag, void, void, void, void>
{
protected:
_Container* container;
public:
typedef _Container container_type;
explicit
back_insert_iterator(_Container& __x)
: container(std::__addressof(__x)) { }
back_insert_iterator&
operator=(const typename _Container::value_type& __value)
{
container->push_back(__value);
return *this;
}
back_insert_iterator&
operator=(typename _Container::value_type&& __value)
{
container->push_back(std::move(__value));
return *this;
}
[[__nodiscard__]]
back_insert_iterator&
operator*()
{ return *this; }
back_insert_iterator&
operator++()
{ return *this; }
back_insert_iterator
operator++(int)
{ return *this; }
};
template<typename _Container>
[[__nodiscard__]]
inline back_insert_iterator<_Container>
back_inserter(_Container& __x)
{ return back_insert_iterator<_Container>(__x); }
template<typename _Container>
class front_insert_iterator
: public iterator<output_iterator_tag, void, void, void, void>
{
protected:
_Container* container;
public:
typedef _Container container_type;
explicit
front_insert_iterator(_Container& __x)
: container(std::__addressof(__x)) { }
front_insert_iterator&
operator=(const typename _Container::value_type& __value)
{
container->push_front(__value);
return *this;
}
front_insert_iterator&
operator=(typename _Container::value_type&& __value)
{
container->push_front(std::move(__value));
return *this;
}
[[__nodiscard__]]
front_insert_iterator&
operator*()
{ return *this; }
front_insert_iterator&
operator++()
{ return *this; }
front_insert_iterator
operator++(int)
{ return *this; }
};
template<typename _Container>
[[__nodiscard__]]
inline front_insert_iterator<_Container>
front_inserter(_Container& __x)
{ return front_insert_iterator<_Container>(__x); }
template<typename _Container>
class insert_iterator
: public iterator<output_iterator_tag, void, void, void, void>
{
typedef typename _Container::iterator _Iter;
protected:
_Container* container;
_Iter iter;
public:
typedef _Container container_type;
insert_iterator(_Container& __x, _Iter __i)
: container(std::__addressof(__x)), iter(__i) {}
insert_iterator&
operator=(const typename _Container::value_type& __value)
{
iter = container->insert(iter, __value);
++iter;
return *this;
}
insert_iterator&
operator=(typename _Container::value_type&& __value)
{
iter = container->insert(iter, std::move(__value));
++iter;
return *this;
}
[[__nodiscard__]]
insert_iterator&
operator*()
{ return *this; }
insert_iterator&
operator++()
{ return *this; }
insert_iterator&
operator++(int)
{ return *this; }
};
#pragma GCC diagnostic pop
template<typename _Container>
[[__nodiscard__]]
inline insert_iterator<_Container>
inserter(_Container& __x, typename _Container::iterator __i)
{ return insert_iterator<_Container>(__x, __i); }
}
namespace __gnu_cxx __attribute__ ((__visibility__ ("default")))
{
template<typename _Iterator, typename _Container>
class __normal_iterator
{
protected:
_Iterator _M_current;
typedef std::iterator_traits<_Iterator> __traits_type;
template<typename _Iter>
using __convertible_from
= std::__enable_if_t<std::is_convertible<_Iter, _Iterator>::value>;
public:
typedef _Iterator iterator_type;
typedef typename __traits_type::iterator_category iterator_category;
typedef typename __traits_type::value_type value_type;
typedef typename __traits_type::difference_type difference_type;
typedef typename __traits_type::reference reference;
typedef typename __traits_type::pointer pointer;
constexpr __normal_iterator() noexcept
: _M_current(_Iterator()) { }
explicit
__normal_iterator(const _Iterator& __i) noexcept
: _M_current(__i) { }
template<typename _Iter, typename = __convertible_from<_Iter>>
__normal_iterator(const __normal_iterator<_Iter, _Container>& __i)
noexcept
: _M_current(__i.base()) { }
reference
operator*() const noexcept
{ return *_M_current; }
pointer
operator->() const noexcept
{ return _M_current; }
__normal_iterator&
operator++() noexcept
{
++_M_current;
return *this;
}
__normal_iterator
operator++(int) noexcept
{ return __normal_iterator(_M_current++); }
__normal_iterator&
operator--() noexcept
{
--_M_current;
return *this;
}
__normal_iterator
operator--(int) noexcept
{ return __normal_iterator(_M_current--); }
reference
operator[](difference_type __n) const noexcept
{ return _M_current[__n]; }
__normal_iterator&
operator+=(difference_type __n) noexcept
{ _M_current += __n; return *this; }
__normal_iterator
operator+(difference_type __n) const noexcept
{ return __normal_iterator(_M_current + __n); }
__normal_iterator&
operator-=(difference_type __n) noexcept
{ _M_current -= __n; return *this; }
__normal_iterator
operator-(difference_type __n) const noexcept
{ return __normal_iterator(_M_current - __n); }
const _Iterator&
base() const noexcept
{ return _M_current; }
};
template<typename _IteratorL, typename _IteratorR, typename _Container>
[[__nodiscard__]]
inline bool
operator==(const __normal_iterator<_IteratorL, _Container>& __lhs,
const __normal_iterator<_IteratorR, _Container>& __rhs)
noexcept
{ return __lhs.base() == __rhs.base(); }
template<typename _Iterator, typename _Container>
[[__nodiscard__]]
inline bool
operator==(const __normal_iterator<_Iterator, _Container>& __lhs,
const __normal_iterator<_Iterator, _Container>& __rhs)
noexcept
{ return __lhs.base() == __rhs.base(); }
template<typename _IteratorL, typename _IteratorR, typename _Container>
[[__nodiscard__]]
inline bool
operator!=(const __normal_iterator<_IteratorL, _Container>& __lhs,
const __normal_iterator<_IteratorR, _Container>& __rhs)
noexcept
{ return __lhs.base() != __rhs.base(); }
template<typename _Iterator, typename _Container>
[[__nodiscard__]]
inline bool
operator!=(const __normal_iterator<_Iterator, _Container>& __lhs,
const __normal_iterator<_Iterator, _Container>& __rhs)
noexcept
{ return __lhs.base() != __rhs.base(); }
template<typename _IteratorL, typename _IteratorR, typename _Container>
[[__nodiscard__]]
inline bool
operator<(const __normal_iterator<_IteratorL, _Container>& __lhs,
const __normal_iterator<_IteratorR, _Container>& __rhs)
noexcept
{ return __lhs.base() < __rhs.base(); }
template<typename _Iterator, typename _Container>
[[__nodiscard__]]
inline bool
operator<(const __normal_iterator<_Iterator, _Container>& __lhs,
const __normal_iterator<_Iterator, _Container>& __rhs)
noexcept
{ return __lhs.base() < __rhs.base(); }
template<typename _IteratorL, typename _IteratorR, typename _Container>
[[__nodiscard__]]
inline bool
operator>(const __normal_iterator<_IteratorL, _Container>& __lhs,
const __normal_iterator<_IteratorR, _Container>& __rhs)
noexcept
{ return __lhs.base() > __rhs.base(); }
template<typename _Iterator, typename _Container>
[[__nodiscard__]]
inline bool
operator>(const __normal_iterator<_Iterator, _Container>& __lhs,
const __normal_iterator<_Iterator, _Container>& __rhs)
noexcept
{ return __lhs.base() > __rhs.base(); }
template<typename _IteratorL, typename _IteratorR, typename _Container>
[[__nodiscard__]]
inline bool
operator<=(const __normal_iterator<_IteratorL, _Container>& __lhs,
const __normal_iterator<_IteratorR, _Container>& __rhs)
noexcept
{ return __lhs.base() <= __rhs.base(); }
template<typename _Iterator, typename _Container>
[[__nodiscard__]]
inline bool
operator<=(const __normal_iterator<_Iterator, _Container>& __lhs,
const __normal_iterator<_Iterator, _Container>& __rhs)
noexcept
{ return __lhs.base() <= __rhs.base(); }
template<typename _IteratorL, typename _IteratorR, typename _Container>
[[__nodiscard__]]
inline bool
operator>=(const __normal_iterator<_IteratorL, _Container>& __lhs,
const __normal_iterator<_IteratorR, _Container>& __rhs)
noexcept
{ return __lhs.base() >= __rhs.base(); }
template<typename _Iterator, typename _Container>
[[__nodiscard__]]
inline bool
operator>=(const __normal_iterator<_Iterator, _Container>& __lhs,
const __normal_iterator<_Iterator, _Container>& __rhs)
noexcept
{ return __lhs.base() >= __rhs.base(); }
template<typename _IteratorL, typename _IteratorR, typename _Container>
[[__nodiscard__]]
inline auto
operator-(const __normal_iterator<_IteratorL, _Container>& __lhs,
const __normal_iterator<_IteratorR, _Container>& __rhs) noexcept
-> decltype(__lhs.base() - __rhs.base())
{ return __lhs.base() - __rhs.base(); }
template<typename _Iterator, typename _Container>
[[__nodiscard__]]
inline typename __normal_iterator<_Iterator, _Container>::difference_type
operator-(const __normal_iterator<_Iterator, _Container>& __lhs,
const __normal_iterator<_Iterator, _Container>& __rhs)
noexcept
{ return __lhs.base() - __rhs.base(); }
template<typename _Iterator, typename _Container>
[[__nodiscard__]]
inline __normal_iterator<_Iterator, _Container>
operator+(typename __normal_iterator<_Iterator, _Container>::difference_type
__n, const __normal_iterator<_Iterator, _Container>& __i)
noexcept
{ return __normal_iterator<_Iterator, _Container>(__i.base() + __n); }
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Iterator, typename _Container>
_Iterator
__niter_base(__gnu_cxx::__normal_iterator<_Iterator, _Container> __it)
noexcept(std::is_nothrow_copy_constructible<_Iterator>::value)
{ return __it.base(); }
template<typename _Iterator, typename _Container>
constexpr auto
__to_address(const __gnu_cxx::__normal_iterator<_Iterator,
_Container>& __it) noexcept
-> decltype(std::__to_address(__it.base()))
{ return std::__to_address(__it.base()); }
namespace __detail
{
}
template<typename _Iterator>
class move_iterator
{
_Iterator _M_current;
using __traits_type = iterator_traits<_Iterator>;
using __base_ref = typename __traits_type::reference;
template<typename _Iter2>
friend class move_iterator;
public:
using iterator_type = _Iterator;
typedef typename __traits_type::iterator_category iterator_category;
typedef typename __traits_type::value_type value_type;
typedef typename __traits_type::difference_type difference_type;
typedef _Iterator pointer;
using reference
= __conditional_t<is_reference<__base_ref>::value,
typename remove_reference<__base_ref>::type&&,
__base_ref>;
constexpr
move_iterator()
: _M_current() { }
explicit constexpr
move_iterator(iterator_type __i)
: _M_current(std::move(__i)) { }
template<typename _Iter>
constexpr
move_iterator(const move_iterator<_Iter>& __i)
: _M_current(__i._M_current) { }
template<typename _Iter>
constexpr
move_iterator& operator=(const move_iterator<_Iter>& __i)
{
_M_current = __i._M_current;
return *this;
}
[[__nodiscard__]]
constexpr iterator_type
base() const
{ return _M_current; }
[[__nodiscard__]]
constexpr reference
operator*() const
{ return static_cast<reference>(*_M_current); }
[[__nodiscard__]]
constexpr pointer
operator->() const
{ return _M_current; }
constexpr move_iterator&
operator++()
{
++_M_current;
return *this;
}
constexpr move_iterator
operator++(int)
{
move_iterator __tmp = *this;
++_M_current;
return __tmp;
}
constexpr move_iterator&
operator--()
{
--_M_current;
return *this;
}
constexpr move_iterator
operator--(int)
{
move_iterator __tmp = *this;
--_M_current;
return __tmp;
}
[[__nodiscard__]]
constexpr move_iterator
operator+(difference_type __n) const
{ return move_iterator(_M_current + __n); }
constexpr move_iterator&
operator+=(difference_type __n)
{
_M_current += __n;
return *this;
}
[[__nodiscard__]]
constexpr move_iterator
operator-(difference_type __n) const
{ return move_iterator(_M_current - __n); }
constexpr move_iterator&
operator-=(difference_type __n)
{
_M_current -= __n;
return *this;
}
[[__nodiscard__]]
constexpr reference
operator[](difference_type __n) const
{ return std::move(_M_current[__n]); }
};
template<typename _IteratorL, typename _IteratorR>
[[__nodiscard__]]
inline constexpr bool
operator==(const move_iterator<_IteratorL>& __x,
const move_iterator<_IteratorR>& __y)
{ return __x.base() == __y.base(); }
template<typename _IteratorL, typename _IteratorR>
[[__nodiscard__]]
inline constexpr bool
operator!=(const move_iterator<_IteratorL>& __x,
const move_iterator<_IteratorR>& __y)
{ return !(__x == __y); }
template<typename _IteratorL, typename _IteratorR>
[[__nodiscard__]]
inline constexpr bool
operator<(const move_iterator<_IteratorL>& __x,
const move_iterator<_IteratorR>& __y)
{ return __x.base() < __y.base(); }
template<typename _IteratorL, typename _IteratorR>
[[__nodiscard__]]
inline constexpr bool
operator<=(const move_iterator<_IteratorL>& __x,
const move_iterator<_IteratorR>& __y)
{ return !(__y < __x); }
template<typename _IteratorL, typename _IteratorR>
[[__nodiscard__]]
inline constexpr bool
operator>(const move_iterator<_IteratorL>& __x,
const move_iterator<_IteratorR>& __y)
{ return __y < __x; }
template<typename _IteratorL, typename _IteratorR>
[[__nodiscard__]]
inline constexpr bool
operator>=(const move_iterator<_IteratorL>& __x,
const move_iterator<_IteratorR>& __y)
{ return !(__x < __y); }
template<typename _Iterator>
[[__nodiscard__]]
inline constexpr bool
operator==(const move_iterator<_Iterator>& __x,
const move_iterator<_Iterator>& __y)
{ return __x.base() == __y.base(); }
template<typename _Iterator>
[[__nodiscard__]]
inline constexpr bool
operator!=(const move_iterator<_Iterator>& __x,
const move_iterator<_Iterator>& __y)
{ return !(__x == __y); }
template<typename _Iterator>
[[__nodiscard__]]
inline constexpr bool
operator<(const move_iterator<_Iterator>& __x,
const move_iterator<_Iterator>& __y)
{ return __x.base() < __y.base(); }
template<typename _Iterator>
[[__nodiscard__]]
inline constexpr bool
operator<=(const move_iterator<_Iterator>& __x,
const move_iterator<_Iterator>& __y)
{ return !(__y < __x); }
template<typename _Iterator>
[[__nodiscard__]]
inline constexpr bool
operator>(const move_iterator<_Iterator>& __x,
const move_iterator<_Iterator>& __y)
{ return __y < __x; }
template<typename _Iterator>
[[__nodiscard__]]
inline constexpr bool
operator>=(const move_iterator<_Iterator>& __x,
const move_iterator<_Iterator>& __y)
{ return !(__x < __y); }
template<typename _IteratorL, typename _IteratorR>
[[__nodiscard__]]
inline constexpr auto
operator-(const move_iterator<_IteratorL>& __x,
const move_iterator<_IteratorR>& __y)
-> decltype(__x.base() - __y.base())
{ return __x.base() - __y.base(); }
template<typename _Iterator>
[[__nodiscard__]]
inline constexpr move_iterator<_Iterator>
operator+(typename move_iterator<_Iterator>::difference_type __n,
const move_iterator<_Iterator>& __x)
{ return __x + __n; }
template<typename _Iterator>
[[__nodiscard__]]
inline constexpr move_iterator<_Iterator>
make_move_iterator(_Iterator __i)
{ return move_iterator<_Iterator>(std::move(__i)); }
template<typename _Iterator, typename _ReturnType
= __conditional_t<__move_if_noexcept_cond
<typename iterator_traits<_Iterator>::value_type>::value,
_Iterator, move_iterator<_Iterator>>>
inline constexpr _ReturnType
__make_move_if_noexcept_iterator(_Iterator __i)
{ return _ReturnType(__i); }
template<typename _Tp, typename _ReturnType
= __conditional_t<__move_if_noexcept_cond<_Tp>::value,
const _Tp*, move_iterator<_Tp*>>>
inline constexpr _ReturnType
__make_move_if_noexcept_iterator(_Tp* __i)
{ return _ReturnType(__i); }
template<typename _Iterator>
auto
__niter_base(move_iterator<_Iterator> __it)
-> decltype(make_move_iterator(__niter_base(__it.base())))
{ return make_move_iterator(__niter_base(__it.base())); }
template<typename _Iterator>
struct __is_move_iterator<move_iterator<_Iterator> >
{
enum { __value = 1 };
typedef __true_type __type;
};
template<typename _Iterator>
auto
__miter_base(move_iterator<_Iterator> __it)
-> decltype(__miter_base(__it.base()))
{ return __miter_base(__it.base()); }
template<typename _InputIterator>
using __iter_key_t = remove_const_t<
typename iterator_traits<_InputIterator>::value_type::first_type>;
template<typename _InputIterator>
using __iter_val_t
= typename iterator_traits<_InputIterator>::value_type::second_type;
template<typename _T1, typename _T2>
struct pair;
template<typename _InputIterator>
using __iter_to_alloc_t
= pair<const __iter_key_t<_InputIterator>, __iter_val_t<_InputIterator>>;
}
namespace std
{
namespace __debug { }
}
namespace __gnu_debug
{
using namespace std::__debug;
template<typename _Ite, typename _Seq, typename _Cat>
struct _Safe_iterator;
}
namespace __gnu_cxx
{
namespace __ops
{
struct _Iter_less_iter
{
template<typename _Iterator1, typename _Iterator2>
constexpr
bool
operator()(_Iterator1 __it1, _Iterator2 __it2) const
{ return *__it1 < *__it2; }
};
constexpr
inline _Iter_less_iter
__iter_less_iter()
{ return _Iter_less_iter(); }
struct _Iter_less_val
{
constexpr _Iter_less_val() = default;
explicit
_Iter_less_val(_Iter_less_iter) { }
template<typename _Iterator, typename _Value>
bool
operator()(_Iterator __it, _Value& __val) const
{ return *__it < __val; }
};
inline _Iter_less_val
__iter_less_val()
{ return _Iter_less_val(); }
inline _Iter_less_val
__iter_comp_val(_Iter_less_iter)
{ return _Iter_less_val(); }
struct _Val_less_iter
{
constexpr _Val_less_iter() = default;
explicit
_Val_less_iter(_Iter_less_iter) { }
template<typename _Value, typename _Iterator>
bool
operator()(_Value& __val, _Iterator __it) const
{ return __val < *__it; }
};
inline _Val_less_iter
__val_less_iter()
{ return _Val_less_iter(); }
inline _Val_less_iter
__val_comp_iter(_Iter_less_iter)
{ return _Val_less_iter(); }
struct _Iter_equal_to_iter
{
template<typename _Iterator1, typename _Iterator2>
bool
operator()(_Iterator1 __it1, _Iterator2 __it2) const
{ return *__it1 == *__it2; }
};
inline _Iter_equal_to_iter
__iter_equal_to_iter()
{ return _Iter_equal_to_iter(); }
struct _Iter_equal_to_val
{
template<typename _Iterator, typename _Value>
bool
operator()(_Iterator __it, _Value& __val) const
{ return *__it == __val; }
};
inline _Iter_equal_to_val
__iter_equal_to_val()
{ return _Iter_equal_to_val(); }
inline _Iter_equal_to_val
__iter_comp_val(_Iter_equal_to_iter)
{ return _Iter_equal_to_val(); }
template<typename _Compare>
struct _Iter_comp_iter
{
_Compare _M_comp;
explicit constexpr
_Iter_comp_iter(_Compare __comp)
: _M_comp(std::move(__comp))
{ }
template<typename _Iterator1, typename _Iterator2>
constexpr
bool
operator()(_Iterator1 __it1, _Iterator2 __it2)
{ return bool(_M_comp(*__it1, *__it2)); }
};
template<typename _Compare>
constexpr
inline _Iter_comp_iter<_Compare>
__iter_comp_iter(_Compare __comp)
{ return _Iter_comp_iter<_Compare>(std::move(__comp)); }
template<typename _Compare>
struct _Iter_comp_val
{
_Compare _M_comp;
explicit
_Iter_comp_val(_Compare __comp)
: _M_comp(std::move(__comp))
{ }
explicit
_Iter_comp_val(const _Iter_comp_iter<_Compare>& __comp)
: _M_comp(__comp._M_comp)
{ }
explicit
_Iter_comp_val(_Iter_comp_iter<_Compare>&& __comp)
: _M_comp(std::move(__comp._M_comp))
{ }
template<typename _Iterator, typename _Value>
bool
operator()(_Iterator __it, _Value& __val)
{ return bool(_M_comp(*__it, __val)); }
};
template<typename _Compare>
inline _Iter_comp_val<_Compare>
__iter_comp_val(_Compare __comp)
{ return _Iter_comp_val<_Compare>(std::move(__comp)); }
template<typename _Compare>
inline _Iter_comp_val<_Compare>
__iter_comp_val(_Iter_comp_iter<_Compare> __comp)
{ return _Iter_comp_val<_Compare>(std::move(__comp)); }
template<typename _Compare>
struct _Val_comp_iter
{
_Compare _M_comp;
explicit
_Val_comp_iter(_Compare __comp)
: _M_comp(std::move(__comp))
{ }
explicit
_Val_comp_iter(const _Iter_comp_iter<_Compare>& __comp)
: _M_comp(__comp._M_comp)
{ }
explicit
_Val_comp_iter(_Iter_comp_iter<_Compare>&& __comp)
: _M_comp(std::move(__comp._M_comp))
{ }
template<typename _Value, typename _Iterator>
bool
operator()(_Value& __val, _Iterator __it)
{ return bool(_M_comp(__val, *__it)); }
};
template<typename _Compare>
inline _Val_comp_iter<_Compare>
__val_comp_iter(_Compare __comp)
{ return _Val_comp_iter<_Compare>(std::move(__comp)); }
template<typename _Compare>
inline _Val_comp_iter<_Compare>
__val_comp_iter(_Iter_comp_iter<_Compare> __comp)
{ return _Val_comp_iter<_Compare>(std::move(__comp)); }
template<typename _Value>
struct _Iter_equals_val
{
_Value& _M_value;
explicit
_Iter_equals_val(_Value& __value)
: _M_value(__value)
{ }
template<typename _Iterator>
bool
operator()(_Iterator __it)
{ return *__it == _M_value; }
};
template<typename _Value>
inline _Iter_equals_val<_Value>
__iter_equals_val(_Value& __val)
{ return _Iter_equals_val<_Value>(__val); }
template<typename _Iterator1>
struct _Iter_equals_iter
{
_Iterator1 _M_it1;
explicit
_Iter_equals_iter(_Iterator1 __it1)
: _M_it1(__it1)
{ }
template<typename _Iterator2>
bool
operator()(_Iterator2 __it2)
{ return *__it2 == *_M_it1; }
};
template<typename _Iterator>
inline _Iter_equals_iter<_Iterator>
__iter_comp_iter(_Iter_equal_to_iter, _Iterator __it)
{ return _Iter_equals_iter<_Iterator>(__it); }
template<typename _Predicate>
struct _Iter_pred
{
_Predicate _M_pred;
explicit
_Iter_pred(_Predicate __pred)
: _M_pred(std::move(__pred))
{ }
template<typename _Iterator>
bool
operator()(_Iterator __it)
{ return bool(_M_pred(*__it)); }
};
template<typename _Predicate>
inline _Iter_pred<_Predicate>
__pred_iter(_Predicate __pred)
{ return _Iter_pred<_Predicate>(std::move(__pred)); }
template<typename _Compare, typename _Value>
struct _Iter_comp_to_val
{
_Compare _M_comp;
_Value& _M_value;
_Iter_comp_to_val(_Compare __comp, _Value& __value)
: _M_comp(std::move(__comp)), _M_value(__value)
{ }
template<typename _Iterator>
bool
operator()(_Iterator __it)
{ return bool(_M_comp(*__it, _M_value)); }
};
template<typename _Compare, typename _Value>
_Iter_comp_to_val<_Compare, _Value>
__iter_comp_val(_Compare __comp, _Value &__val)
{
return _Iter_comp_to_val<_Compare, _Value>(std::move(__comp), __val);
}
template<typename _Compare, typename _Iterator1>
struct _Iter_comp_to_iter
{
_Compare _M_comp;
_Iterator1 _M_it1;
_Iter_comp_to_iter(_Compare __comp, _Iterator1 __it1)
: _M_comp(std::move(__comp)), _M_it1(__it1)
{ }
template<typename _Iterator2>
bool
operator()(_Iterator2 __it2)
{ return bool(_M_comp(*__it2, *_M_it1)); }
};
template<typename _Compare, typename _Iterator>
inline _Iter_comp_to_iter<_Compare, _Iterator>
__iter_comp_iter(_Iter_comp_iter<_Compare> __comp, _Iterator __it)
{
return _Iter_comp_to_iter<_Compare, _Iterator>(
std::move(__comp._M_comp), __it);
}
template<typename _Predicate>
struct _Iter_negate
{
_Predicate _M_pred;
explicit
_Iter_negate(_Predicate __pred)
: _M_pred(std::move(__pred))
{ }
template<typename _Iterator>
bool
operator()(_Iterator __it)
{ return !bool(_M_pred(*__it)); }
};
template<typename _Predicate>
inline _Iter_negate<_Predicate>
__negate(_Iter_pred<_Predicate> __pred)
{ return _Iter_negate<_Predicate>(std::move(__pred._M_pred)); }
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Tp>
constexpr _Tp
__rotl(_Tp __x, int __s) noexcept
{
constexpr auto _Nd = __gnu_cxx::__int_traits<_Tp>::__digits;
if constexpr ((_Nd & (_Nd - 1)) == 0)
{
constexpr unsigned __uNd = _Nd;
const unsigned __r = __s;
return (__x << (__r % __uNd)) | (__x >> ((-__r) % __uNd));
}
const int __r = __s % _Nd;
if (__r == 0)
return __x;
else if (__r > 0)
return (__x << __r) | (__x >> ((_Nd - __r) % _Nd));
else
return (__x >> -__r) | (__x << ((_Nd + __r) % _Nd));
}
template<typename _Tp>
constexpr _Tp
__rotr(_Tp __x, int __s) noexcept
{
constexpr auto _Nd = __gnu_cxx::__int_traits<_Tp>::__digits;
if constexpr ((_Nd & (_Nd - 1)) == 0)
{
constexpr unsigned __uNd = _Nd;
const unsigned __r = __s;
return (__x >> (__r % __uNd)) | (__x << ((-__r) % __uNd));
}
const int __r = __s % _Nd;
if (__r == 0)
return __x;
else if (__r > 0)
return (__x >> __r) | (__x << ((_Nd - __r) % _Nd));
else
return (__x << -__r) | (__x >> ((_Nd + __r) % _Nd));
}
template<typename _Tp>
constexpr int
__countl_zero(_Tp __x) noexcept
{
using __gnu_cxx::__int_traits;
constexpr auto _Nd = __int_traits<_Tp>::__digits;
if (__x == 0)
return _Nd;
constexpr auto _Nd_ull = __int_traits<unsigned long long>::__digits;
constexpr auto _Nd_ul = __int_traits<unsigned long>::__digits;
constexpr auto _Nd_u = __int_traits<unsigned>::__digits;
if constexpr (_Nd <= _Nd_u)
{
constexpr int __diff = _Nd_u - _Nd;
return __builtin_clz(__x) - __diff;
}
else if constexpr (_Nd <= _Nd_ul)
{
constexpr int __diff = _Nd_ul - _Nd;
return __builtin_clzl(__x) - __diff;
}
else if constexpr (_Nd <= _Nd_ull)
{
constexpr int __diff = _Nd_ull - _Nd;
return __builtin_clzll(__x) - __diff;
}
else
{
static_assert(_Nd <= (2 * _Nd_ull),
"Maximum supported integer size is 128-bit");
unsigned long long __high = __x >> _Nd_ull;
if (__high != 0)
{
constexpr int __diff = (2 * _Nd_ull) - _Nd;
return __builtin_clzll(__high) - __diff;
}
constexpr auto __max_ull = __int_traits<unsigned long long>::__max;
unsigned long long __low = __x & __max_ull;
return (_Nd - _Nd_ull) + __builtin_clzll(__low);
}
}
template<typename _Tp>
constexpr int
__countl_one(_Tp __x) noexcept
{
return std::__countl_zero<_Tp>((_Tp)~__x);
}
template<typename _Tp>
constexpr int
__countr_zero(_Tp __x) noexcept
{
using __gnu_cxx::__int_traits;
constexpr auto _Nd = __int_traits<_Tp>::__digits;
if (__x == 0)
return _Nd;
constexpr auto _Nd_ull = __int_traits<unsigned long long>::__digits;
constexpr auto _Nd_ul = __int_traits<unsigned long>::__digits;
constexpr auto _Nd_u = __int_traits<unsigned>::__digits;
if constexpr (_Nd <= _Nd_u)
return __builtin_ctz(__x);
else if constexpr (_Nd <= _Nd_ul)
return __builtin_ctzl(__x);
else if constexpr (_Nd <= _Nd_ull)
return __builtin_ctzll(__x);
else
{
static_assert(_Nd <= (2 * _Nd_ull),
"Maximum supported integer size is 128-bit");
constexpr auto __max_ull = __int_traits<unsigned long long>::__max;
unsigned long long __low = __x & __max_ull;
if (__low != 0)
return __builtin_ctzll(__low);
unsigned long long __high = __x >> _Nd_ull;
return __builtin_ctzll(__high) + _Nd_ull;
}
}
template<typename _Tp>
constexpr int
__countr_one(_Tp __x) noexcept
{
return std::__countr_zero((_Tp)~__x);
}
template<typename _Tp>
constexpr int
__popcount(_Tp __x) noexcept
{
using __gnu_cxx::__int_traits;
constexpr auto _Nd = __int_traits<_Tp>::__digits;
constexpr auto _Nd_ull = __int_traits<unsigned long long>::__digits;
constexpr auto _Nd_ul = __int_traits<unsigned long>::__digits;
constexpr auto _Nd_u = __int_traits<unsigned>::__digits;
if constexpr (_Nd <= _Nd_u)
return __builtin_popcount(__x);
else if constexpr (_Nd <= _Nd_ul)
return __builtin_popcountl(__x);
else if constexpr (_Nd <= _Nd_ull)
return __builtin_popcountll(__x);
else
{
static_assert(_Nd <= (2 * _Nd_ull),
"Maximum supported integer size is 128-bit");
constexpr auto __max_ull = __int_traits<unsigned long long>::__max;
unsigned long long __low = __x & __max_ull;
unsigned long long __high = __x >> _Nd_ull;
return __builtin_popcountll(__low) + __builtin_popcountll(__high);
}
}
template<typename _Tp>
constexpr bool
__has_single_bit(_Tp __x) noexcept
{ return std::__popcount(__x) == 1; }
template<typename _Tp>
constexpr _Tp
__bit_ceil(_Tp __x) noexcept
{
using __gnu_cxx::__int_traits;
constexpr auto _Nd = __int_traits<_Tp>::__digits;
if (__x == 0 || __x == 1)
return 1;
auto __shift_exponent = _Nd - std::__countl_zero((_Tp)(__x - 1u));
if (!std::__is_constant_evaluated())
{
do { if (std::__is_constant_evaluated() && !bool(__shift_exponent != __int_traits<_Tp>::__digits)) __builtin_unreachable(); } while (false);
}
using __promoted_type = decltype(__x << 1);
if constexpr (!is_same<__promoted_type, _Tp>::value)
{
const int __extra_exp = sizeof(__promoted_type) / sizeof(_Tp) / 2;
__shift_exponent |= (__shift_exponent & _Nd) << __extra_exp;
}
return (_Tp)1u << __shift_exponent;
}
template<typename _Tp>
constexpr _Tp
__bit_floor(_Tp __x) noexcept
{
constexpr auto _Nd = __gnu_cxx::__int_traits<_Tp>::__digits;
if (__x == 0)
return 0;
return (_Tp)1u << (_Nd - std::__countl_zero((_Tp)(__x >> 1)));
}
template<typename _Tp>
constexpr int
__bit_width(_Tp __x) noexcept
{
constexpr auto _Nd = __gnu_cxx::__int_traits<_Tp>::__digits;
return _Nd - std::__countl_zero(__x);
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Tp, typename _Up>
constexpr
inline int
__memcmp(const _Tp* __first1, const _Up* __first2, size_t __num)
{
static_assert(sizeof(_Tp) == sizeof(_Up), "can be compared with memcmp");
return __builtin_memcmp(__first1, __first2, sizeof(_Tp) * __num);
}
template<typename _ForwardIterator1, typename _ForwardIterator2>
inline void
iter_swap(_ForwardIterator1 __a, _ForwardIterator2 __b)
{
swap(*__a, *__b);
}
template<typename _ForwardIterator1, typename _ForwardIterator2>
_ForwardIterator2
swap_ranges(_ForwardIterator1 __first1, _ForwardIterator1 __last1,
_ForwardIterator2 __first2)
{
;
for (; __first1 != __last1; ++__first1, (void)++__first2)
std::iter_swap(__first1, __first2);
return __first2;
}
template<typename _Tp>
constexpr
inline const _Tp&
min(const _Tp& __a, const _Tp& __b)
{
if (__b < __a)
return __b;
return __a;
}
template<typename _Tp>
constexpr
inline const _Tp&
max(const _Tp& __a, const _Tp& __b)
{
if (__a < __b)
return __b;
return __a;
}
template<typename _Tp, typename _Compare>
constexpr
inline const _Tp&
min(const _Tp& __a, const _Tp& __b, _Compare __comp)
{
if (__comp(__b, __a))
return __b;
return __a;
}
template<typename _Tp, typename _Compare>
constexpr
inline const _Tp&
max(const _Tp& __a, const _Tp& __b, _Compare __comp)
{
if (__comp(__a, __b))
return __b;
return __a;
}
template<typename _Iterator>
inline _Iterator
__niter_base(_Iterator __it)
noexcept(std::is_nothrow_copy_constructible<_Iterator>::value)
{ return __it; }
template<typename _Ite, typename _Seq>
_Ite
__niter_base(const ::__gnu_debug::_Safe_iterator<_Ite, _Seq,
std::random_access_iterator_tag>&);
template<typename _From, typename _To>
inline _From
__niter_wrap(_From __from, _To __res)
{ return __from + (__res - std::__niter_base(__from)); }
template<typename _Iterator>
inline _Iterator
__niter_wrap(const _Iterator&, _Iterator __res)
{ return __res; }
template<bool _IsMove, bool _IsSimple, typename _Category>
struct __copy_move
{
template<typename _II, typename _OI>
static _OI
__copy_m(_II __first, _II __last, _OI __result)
{
for (; __first != __last; ++__result, (void)++__first)
*__result = *__first;
return __result;
}
};
template<typename _Category>
struct __copy_move<true, false, _Category>
{
template<typename _II, typename _OI>
static _OI
__copy_m(_II __first, _II __last, _OI __result)
{
for (; __first != __last; ++__result, (void)++__first)
*__result = std::move(*__first);
return __result;
}
};
template<>
struct __copy_move<false, false, random_access_iterator_tag>
{
template<typename _II, typename _OI>
static _OI
__copy_m(_II __first, _II __last, _OI __result)
{
typedef typename iterator_traits<_II>::difference_type _Distance;
for(_Distance __n = __last - __first; __n > 0; --__n)
{
*__result = *__first;
++__first;
++__result;
}
return __result;
}
template<typename _Tp, typename _Up>
static void
__assign_one(_Tp* __to, _Up* __from)
{ *__to = *__from; }
};
template<>
struct __copy_move<true, false, random_access_iterator_tag>
{
template<typename _II, typename _OI>
static _OI
__copy_m(_II __first, _II __last, _OI __result)
{
typedef typename iterator_traits<_II>::difference_type _Distance;
for(_Distance __n = __last - __first; __n > 0; --__n)
{
*__result = std::move(*__first);
++__first;
++__result;
}
return __result;
}
template<typename _Tp, typename _Up>
static void
__assign_one(_Tp* __to, _Up* __from)
{ *__to = std::move(*__from); }
};
template<bool _IsMove>
struct __copy_move<_IsMove, true, random_access_iterator_tag>
{
template<typename _Tp, typename _Up>
static _Up*
__copy_m(_Tp* __first, _Tp* __last, _Up* __result)
{
const ptrdiff_t _Num = __last - __first;
if (__builtin_expect(_Num > 1, true))
__builtin_memmove(__result, __first, sizeof(_Tp) * _Num);
else if (_Num == 1)
std::__copy_move<_IsMove, false, random_access_iterator_tag>::
__assign_one(__result, __first);
return __result + _Num;
}
};
template<typename _Tp, typename _Ref, typename _Ptr>
struct _Deque_iterator;
struct _Bit_iterator;
template<typename _CharT>
struct char_traits;
template<typename _CharT, typename _Traits>
class istreambuf_iterator;
template<typename _CharT, typename _Traits>
class ostreambuf_iterator;
template<bool _IsMove, typename _CharT>
typename __gnu_cxx::__enable_if<__is_char<_CharT>::__value,
ostreambuf_iterator<_CharT, char_traits<_CharT> > >::__type
__copy_move_a2(_CharT*, _CharT*,
ostreambuf_iterator<_CharT, char_traits<_CharT> >);
template<bool _IsMove, typename _CharT>
typename __gnu_cxx::__enable_if<__is_char<_CharT>::__value,
ostreambuf_iterator<_CharT, char_traits<_CharT> > >::__type
__copy_move_a2(const _CharT*, const _CharT*,
ostreambuf_iterator<_CharT, char_traits<_CharT> >);
template<bool _IsMove, typename _CharT>
typename __gnu_cxx::__enable_if<__is_char<_CharT>::__value,
_CharT*>::__type
__copy_move_a2(istreambuf_iterator<_CharT, char_traits<_CharT> >,
istreambuf_iterator<_CharT, char_traits<_CharT> >, _CharT*);
template<bool _IsMove, typename _CharT>
typename __gnu_cxx::__enable_if<
__is_char<_CharT>::__value,
std::_Deque_iterator<_CharT, _CharT&, _CharT*> >::__type
__copy_move_a2(
istreambuf_iterator<_CharT, char_traits<_CharT> >,
istreambuf_iterator<_CharT, char_traits<_CharT> >,
std::_Deque_iterator<_CharT, _CharT&, _CharT*>);
template<bool _IsMove, typename _II, typename _OI>
inline _OI
__copy_move_a2(_II __first, _II __last, _OI __result)
{
typedef typename iterator_traits<_II>::iterator_category _Category;
return std::__copy_move<_IsMove, __memcpyable<_OI, _II>::__value,
_Category>::__copy_m(__first, __last, __result);
}
template<bool _IsMove,
typename _Tp, typename _Ref, typename _Ptr, typename _OI>
_OI
__copy_move_a1(std::_Deque_iterator<_Tp, _Ref, _Ptr>,
std::_Deque_iterator<_Tp, _Ref, _Ptr>,
_OI);
template<bool _IsMove,
typename _ITp, typename _IRef, typename _IPtr, typename _OTp>
std::_Deque_iterator<_OTp, _OTp&, _OTp*>
__copy_move_a1(std::_Deque_iterator<_ITp, _IRef, _IPtr>,
std::_Deque_iterator<_ITp, _IRef, _IPtr>,
std::_Deque_iterator<_OTp, _OTp&, _OTp*>);
template<bool _IsMove, typename _II, typename _Tp>
typename __gnu_cxx::__enable_if<
__is_random_access_iter<_II>::__value,
std::_Deque_iterator<_Tp, _Tp&, _Tp*> >::__type
__copy_move_a1(_II, _II, std::_Deque_iterator<_Tp, _Tp&, _Tp*>);
template<bool _IsMove, typename _II, typename _OI>
inline _OI
__copy_move_a1(_II __first, _II __last, _OI __result)
{ return std::__copy_move_a2<_IsMove>(__first, __last, __result); }
template<bool _IsMove, typename _II, typename _OI>
inline _OI
__copy_move_a(_II __first, _II __last, _OI __result)
{
return std::__niter_wrap(__result,
std::__copy_move_a1<_IsMove>(std::__niter_base(__first),
std::__niter_base(__last),
std::__niter_base(__result)));
}
template<bool _IsMove,
typename _Ite, typename _Seq, typename _Cat, typename _OI>
_OI
__copy_move_a(const ::__gnu_debug::_Safe_iterator<_Ite, _Seq, _Cat>&,
const ::__gnu_debug::_Safe_iterator<_Ite, _Seq, _Cat>&,
_OI);
template<bool _IsMove,
typename _II, typename _Ite, typename _Seq, typename _Cat>
__gnu_debug::_Safe_iterator<_Ite, _Seq, _Cat>
__copy_move_a(_II, _II,
const ::__gnu_debug::_Safe_iterator<_Ite, _Seq, _Cat>&);
template<bool _IsMove,
typename _IIte, typename _ISeq, typename _ICat,
typename _OIte, typename _OSeq, typename _OCat>
::__gnu_debug::_Safe_iterator<_OIte, _OSeq, _OCat>
__copy_move_a(const ::__gnu_debug::_Safe_iterator<_IIte, _ISeq, _ICat>&,
const ::__gnu_debug::_Safe_iterator<_IIte, _ISeq, _ICat>&,
const ::__gnu_debug::_Safe_iterator<_OIte, _OSeq, _OCat>&);
template<typename _InputIterator, typename _Size, typename _OutputIterator>
_OutputIterator
__copy_n_a(_InputIterator __first, _Size __n, _OutputIterator __result,
bool)
{
if (__n > 0)
{
while (true)
{
*__result = *__first;
++__result;
if (--__n > 0)
++__first;
else
break;
}
}
return __result;
}
template<typename _CharT, typename _Size>
typename __gnu_cxx::__enable_if<
__is_char<_CharT>::__value, _CharT*>::__type
__copy_n_a(istreambuf_iterator<_CharT, char_traits<_CharT> >,
_Size, _CharT*, bool);
template<typename _CharT, typename _Size>
typename __gnu_cxx::__enable_if<
__is_char<_CharT>::__value,
std::_Deque_iterator<_CharT, _CharT&, _CharT*> >::__type
__copy_n_a(istreambuf_iterator<_CharT, char_traits<_CharT> >, _Size,
std::_Deque_iterator<_CharT, _CharT&, _CharT*>,
bool);
template<typename _II, typename _OI>
inline _OI
copy(_II __first, _II __last, _OI __result)
{
;
return std::__copy_move_a<__is_move_iterator<_II>::__value>
(std::__miter_base(__first), std::__miter_base(__last), __result);
}
template<typename _II, typename _OI>
inline _OI
move(_II __first, _II __last, _OI __result)
{
;
return std::__copy_move_a<true>(std::__miter_base(__first),
std::__miter_base(__last), __result);
}
template<bool _IsMove, bool _IsSimple, typename _Category>
struct __copy_move_backward
{
template<typename _BI1, typename _BI2>
static _BI2
__copy_move_b(_BI1 __first, _BI1 __last, _BI2 __result)
{
while (__first != __last)
*--__result = *--__last;
return __result;
}
};
template<typename _Category>
struct __copy_move_backward<true, false, _Category>
{
template<typename _BI1, typename _BI2>
static _BI2
__copy_move_b(_BI1 __first, _BI1 __last, _BI2 __result)
{
while (__first != __last)
*--__result = std::move(*--__last);
return __result;
}
};
template<>
struct __copy_move_backward<false, false, random_access_iterator_tag>
{
template<typename _BI1, typename _BI2>
static _BI2
__copy_move_b(_BI1 __first, _BI1 __last, _BI2 __result)
{
typename iterator_traits<_BI1>::difference_type
__n = __last - __first;
for (; __n > 0; --__n)
*--__result = *--__last;
return __result;
}
};
template<>
struct __copy_move_backward<true, false, random_access_iterator_tag>
{
template<typename _BI1, typename _BI2>
static _BI2
__copy_move_b(_BI1 __first, _BI1 __last, _BI2 __result)
{
typename iterator_traits<_BI1>::difference_type
__n = __last - __first;
for (; __n > 0; --__n)
*--__result = std::move(*--__last);
return __result;
}
};
template<bool _IsMove>
struct __copy_move_backward<_IsMove, true, random_access_iterator_tag>
{
template<typename _Tp, typename _Up>
static _Up*
__copy_move_b(_Tp* __first, _Tp* __last, _Up* __result)
{
const ptrdiff_t _Num = __last - __first;
if (__builtin_expect(_Num > 1, true))
__builtin_memmove(__result - _Num, __first, sizeof(_Tp) * _Num);
else if (_Num == 1)
std::__copy_move<_IsMove, false, random_access_iterator_tag>::
__assign_one(__result - 1, __first);
return __result - _Num;
}
};
template<bool _IsMove, typename _BI1, typename _BI2>
inline _BI2
__copy_move_backward_a2(_BI1 __first, _BI1 __last, _BI2 __result)
{
typedef typename iterator_traits<_BI1>::iterator_category _Category;
return std::__copy_move_backward<_IsMove,
__memcpyable<_BI2, _BI1>::__value,
_Category>::__copy_move_b(__first,
__last,
__result);
}
template<bool _IsMove, typename _BI1, typename _BI2>
inline _BI2
__copy_move_backward_a1(_BI1 __first, _BI1 __last, _BI2 __result)
{ return std::__copy_move_backward_a2<_IsMove>(__first, __last, __result); }
template<bool _IsMove,
typename _Tp, typename _Ref, typename _Ptr, typename _OI>
_OI
__copy_move_backward_a1(std::_Deque_iterator<_Tp, _Ref, _Ptr>,
std::_Deque_iterator<_Tp, _Ref, _Ptr>,
_OI);
template<bool _IsMove,
typename _ITp, typename _IRef, typename _IPtr, typename _OTp>
std::_Deque_iterator<_OTp, _OTp&, _OTp*>
__copy_move_backward_a1(
std::_Deque_iterator<_ITp, _IRef, _IPtr>,
std::_Deque_iterator<_ITp, _IRef, _IPtr>,
std::_Deque_iterator<_OTp, _OTp&, _OTp*>);
template<bool _IsMove, typename _II, typename _Tp>
typename __gnu_cxx::__enable_if<
__is_random_access_iter<_II>::__value,
std::_Deque_iterator<_Tp, _Tp&, _Tp*> >::__type
__copy_move_backward_a1(_II, _II,
std::_Deque_iterator<_Tp, _Tp&, _Tp*>);
template<bool _IsMove, typename _II, typename _OI>
inline _OI
__copy_move_backward_a(_II __first, _II __last, _OI __result)
{
return std::__niter_wrap(__result,
std::__copy_move_backward_a1<_IsMove>
(std::__niter_base(__first), std::__niter_base(__last),
std::__niter_base(__result)));
}
template<bool _IsMove,
typename _Ite, typename _Seq, typename _Cat, typename _OI>
_OI
__copy_move_backward_a(
const ::__gnu_debug::_Safe_iterator<_Ite, _Seq, _Cat>&,
const ::__gnu_debug::_Safe_iterator<_Ite, _Seq, _Cat>&,
_OI);
template<bool _IsMove,
typename _II, typename _Ite, typename _Seq, typename _Cat>
__gnu_debug::_Safe_iterator<_Ite, _Seq, _Cat>
__copy_move_backward_a(_II, _II,
const ::__gnu_debug::_Safe_iterator<_Ite, _Seq, _Cat>&);
template<bool _IsMove,
typename _IIte, typename _ISeq, typename _ICat,
typename _OIte, typename _OSeq, typename _OCat>
::__gnu_debug::_Safe_iterator<_OIte, _OSeq, _OCat>
__copy_move_backward_a(
const ::__gnu_debug::_Safe_iterator<_IIte, _ISeq, _ICat>&,
const ::__gnu_debug::_Safe_iterator<_IIte, _ISeq, _ICat>&,
const ::__gnu_debug::_Safe_iterator<_OIte, _OSeq, _OCat>&);
template<typename _BI1, typename _BI2>
inline _BI2
copy_backward(_BI1 __first, _BI1 __last, _BI2 __result)
{
;
return std::__copy_move_backward_a<__is_move_iterator<_BI1>::__value>
(std::__miter_base(__first), std::__miter_base(__last), __result);
}
template<typename _BI1, typename _BI2>
inline _BI2
move_backward(_BI1 __first, _BI1 __last, _BI2 __result)
{
;
return std::__copy_move_backward_a<true>(std::__miter_base(__first),
std::__miter_base(__last),
__result);
}
template<typename _ForwardIterator, typename _Tp>
inline typename
__gnu_cxx::__enable_if<!__is_scalar<_Tp>::__value, void>::__type
__fill_a1(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __value)
{
for (; __first != __last; ++__first)
*__first = __value;
}
template<typename _ForwardIterator, typename _Tp>
inline typename
__gnu_cxx::__enable_if<__is_scalar<_Tp>::__value, void>::__type
__fill_a1(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __value)
{
const _Tp __tmp = __value;
for (; __first != __last; ++__first)
*__first = __tmp;
}
template<typename _Tp>
inline typename
__gnu_cxx::__enable_if<__is_byte<_Tp>::__value, void>::__type
__fill_a1(_Tp* __first, _Tp* __last, const _Tp& __c)
{
const _Tp __tmp = __c;
if (const size_t __len = __last - __first)
__builtin_memset(__first, static_cast<unsigned char>(__tmp), __len);
}
template<typename _Ite, typename _Cont, typename _Tp>
inline void
__fill_a1(::__gnu_cxx::__normal_iterator<_Ite, _Cont> __first,
::__gnu_cxx::__normal_iterator<_Ite, _Cont> __last,
const _Tp& __value)
{ std::__fill_a1(__first.base(), __last.base(), __value); }
template<typename _Tp, typename _VTp>
void
__fill_a1(const std::_Deque_iterator<_Tp, _Tp&, _Tp*>&,
const std::_Deque_iterator<_Tp, _Tp&, _Tp*>&,
const _VTp&);
void
__fill_a1(std::_Bit_iterator, std::_Bit_iterator,
const bool&);
template<typename _FIte, typename _Tp>
inline void
__fill_a(_FIte __first, _FIte __last, const _Tp& __value)
{ std::__fill_a1(__first, __last, __value); }
template<typename _Ite, typename _Seq, typename _Cat, typename _Tp>
void
__fill_a(const ::__gnu_debug::_Safe_iterator<_Ite, _Seq, _Cat>&,
const ::__gnu_debug::_Safe_iterator<_Ite, _Seq, _Cat>&,
const _Tp&);
template<typename _ForwardIterator, typename _Tp>
inline void
fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value)
{
;
std::__fill_a(__first, __last, __value);
}
inline constexpr int
__size_to_integer(int __n) { return __n; }
inline constexpr unsigned
__size_to_integer(unsigned __n) { return __n; }
inline constexpr long
__size_to_integer(long __n) { return __n; }
inline constexpr unsigned long
__size_to_integer(unsigned long __n) { return __n; }
inline constexpr long long
__size_to_integer(long long __n) { return __n; }
inline constexpr unsigned long long
__size_to_integer(unsigned long long __n) { return __n; }
inline constexpr long long
__size_to_integer(float __n) { return (long long)__n; }
inline constexpr long long
__size_to_integer(double __n) { return (long long)__n; }
inline constexpr long long
__size_to_integer(long double __n) { return (long long)__n; }
__extension__ inline constexpr long long
__size_to_integer(__float128 __n) { return (long long)__n; }
template<typename _OutputIterator, typename _Size, typename _Tp>
inline typename
__gnu_cxx::__enable_if<!__is_scalar<_Tp>::__value, _OutputIterator>::__type
__fill_n_a1(_OutputIterator __first, _Size __n, const _Tp& __value)
{
for (; __n > 0; --__n, (void) ++__first)
*__first = __value;
return __first;
}
template<typename _OutputIterator, typename _Size, typename _Tp>
inline typename
__gnu_cxx::__enable_if<__is_scalar<_Tp>::__value, _OutputIterator>::__type
__fill_n_a1(_OutputIterator __first, _Size __n, const _Tp& __value)
{
const _Tp __tmp = __value;
for (; __n > 0; --__n, (void) ++__first)
*__first = __tmp;
return __first;
}
template<typename _Ite, typename _Seq, typename _Cat, typename _Size,
typename _Tp>
::__gnu_debug::_Safe_iterator<_Ite, _Seq, _Cat>
__fill_n_a(const ::__gnu_debug::_Safe_iterator<_Ite, _Seq, _Cat>& __first,
_Size __n, const _Tp& __value,
std::input_iterator_tag);
template<typename _OutputIterator, typename _Size, typename _Tp>
inline _OutputIterator
__fill_n_a(_OutputIterator __first, _Size __n, const _Tp& __value,
std::output_iterator_tag)
{
static_assert(is_integral<_Size>{}, "fill_n must pass integral size");
return __fill_n_a1(__first, __n, __value);
}
template<typename _OutputIterator, typename _Size, typename _Tp>
inline _OutputIterator
__fill_n_a(_OutputIterator __first, _Size __n, const _Tp& __value,
std::input_iterator_tag)
{
static_assert(is_integral<_Size>{}, "fill_n must pass integral size");
return __fill_n_a1(__first, __n, __value);
}
template<typename _OutputIterator, typename _Size, typename _Tp>
inline _OutputIterator
__fill_n_a(_OutputIterator __first, _Size __n, const _Tp& __value,
std::random_access_iterator_tag)
{
static_assert(is_integral<_Size>{}, "fill_n must pass integral size");
if (__n <= 0)
return __first;
;
std::__fill_a(__first, __first + __n, __value);
return __first + __n;
}
template<typename _OI, typename _Size, typename _Tp>
inline _OI
fill_n(_OI __first, _Size __n, const _Tp& __value)
{
return std::__fill_n_a(__first, std::__size_to_integer(__n), __value,
std::__iterator_category(__first));
}
template<bool _BoolType>
struct __equal
{
template<typename _II1, typename _II2>
static bool
equal(_II1 __first1, _II1 __last1, _II2 __first2)
{
for (; __first1 != __last1; ++__first1, (void) ++__first2)
if (!(*__first1 == *__first2))
return false;
return true;
}
};
template<>
struct __equal<true>
{
template<typename _Tp>
static bool
equal(const _Tp* __first1, const _Tp* __last1, const _Tp* __first2)
{
if (const size_t __len = (__last1 - __first1))
return !std::__memcmp(__first1, __first2, __len);
return true;
}
};
template<typename _Tp, typename _Ref, typename _Ptr, typename _II>
typename __gnu_cxx::__enable_if<
__is_random_access_iter<_II>::__value, bool>::__type
__equal_aux1(std::_Deque_iterator<_Tp, _Ref, _Ptr>,
std::_Deque_iterator<_Tp, _Ref, _Ptr>,
_II);
template<typename _Tp1, typename _Ref1, typename _Ptr1,
typename _Tp2, typename _Ref2, typename _Ptr2>
bool
__equal_aux1(std::_Deque_iterator<_Tp1, _Ref1, _Ptr1>,
std::_Deque_iterator<_Tp1, _Ref1, _Ptr1>,
std::_Deque_iterator<_Tp2, _Ref2, _Ptr2>);
template<typename _II, typename _Tp, typename _Ref, typename _Ptr>
typename __gnu_cxx::__enable_if<
__is_random_access_iter<_II>::__value, bool>::__type
__equal_aux1(_II, _II,
std::_Deque_iterator<_Tp, _Ref, _Ptr>);
template<typename _II1, typename _II2>
inline bool
__equal_aux1(_II1 __first1, _II1 __last1, _II2 __first2)
{
typedef typename iterator_traits<_II1>::value_type _ValueType1;
const bool __simple = ((__is_integer<_ValueType1>::__value
|| __is_pointer<_ValueType1>::__value)
&& __memcmpable<_II1, _II2>::__value);
return std::__equal<__simple>::equal(__first1, __last1, __first2);
}
template<typename _II1, typename _II2>
inline bool
__equal_aux(_II1 __first1, _II1 __last1, _II2 __first2)
{
return std::__equal_aux1(std::__niter_base(__first1),
std::__niter_base(__last1),
std::__niter_base(__first2));
}
template<typename _II1, typename _Seq1, typename _Cat1, typename _II2>
bool
__equal_aux(const ::__gnu_debug::_Safe_iterator<_II1, _Seq1, _Cat1>&,
const ::__gnu_debug::_Safe_iterator<_II1, _Seq1, _Cat1>&,
_II2);
template<typename _II1, typename _II2, typename _Seq2, typename _Cat2>
bool
__equal_aux(_II1, _II1,
const ::__gnu_debug::_Safe_iterator<_II2, _Seq2, _Cat2>&);
template<typename _II1, typename _Seq1, typename _Cat1,
typename _II2, typename _Seq2, typename _Cat2>
bool
__equal_aux(const ::__gnu_debug::_Safe_iterator<_II1, _Seq1, _Cat1>&,
const ::__gnu_debug::_Safe_iterator<_II1, _Seq1, _Cat1>&,
const ::__gnu_debug::_Safe_iterator<_II2, _Seq2, _Cat2>&);
template<typename, typename>
struct __lc_rai
{
template<typename _II1, typename _II2>
static _II1
__newlast1(_II1, _II1 __last1, _II2, _II2)
{ return __last1; }
template<typename _II>
static bool
__cnd2(_II __first, _II __last)
{ return __first != __last; }
};
template<>
struct __lc_rai<random_access_iterator_tag, random_access_iterator_tag>
{
template<typename _RAI1, typename _RAI2>
static _RAI1
__newlast1(_RAI1 __first1, _RAI1 __last1,
_RAI2 __first2, _RAI2 __last2)
{
const typename iterator_traits<_RAI1>::difference_type
__diff1 = __last1 - __first1;
const typename iterator_traits<_RAI2>::difference_type
__diff2 = __last2 - __first2;
return __diff2 < __diff1 ? __first1 + __diff2 : __last1;
}
template<typename _RAI>
static bool
__cnd2(_RAI, _RAI)
{ return true; }
};
template<typename _II1, typename _II2, typename _Compare>
bool
__lexicographical_compare_impl(_II1 __first1, _II1 __last1,
_II2 __first2, _II2 __last2,
_Compare __comp)
{
typedef typename iterator_traits<_II1>::iterator_category _Category1;
typedef typename iterator_traits<_II2>::iterator_category _Category2;
typedef std::__lc_rai<_Category1, _Category2> __rai_type;
__last1 = __rai_type::__newlast1(__first1, __last1, __first2, __last2);
for (; __first1 != __last1 && __rai_type::__cnd2(__first2, __last2);
++__first1, (void)++__first2)
{
if (__comp(__first1, __first2))
return true;
if (__comp(__first2, __first1))
return false;
}
return __first1 == __last1 && __first2 != __last2;
}
template<bool _BoolType>
struct __lexicographical_compare
{
template<typename _II1, typename _II2>
static bool
__lc(_II1 __first1, _II1 __last1, _II2 __first2, _II2 __last2)
{
using __gnu_cxx::__ops::__iter_less_iter;
return std::__lexicographical_compare_impl(__first1, __last1,
__first2, __last2,
__iter_less_iter());
}
template<typename _II1, typename _II2>
static int
__3way(_II1 __first1, _II1 __last1, _II2 __first2, _II2 __last2)
{
while (__first1 != __last1)
{
if (__first2 == __last2)
return +1;
if (*__first1 < *__first2)
return -1;
if (*__first2 < *__first1)
return +1;
++__first1;
++__first2;
}
return int(__first2 == __last2) - 1;
}
};
template<>
struct __lexicographical_compare<true>
{
template<typename _Tp, typename _Up>
static bool
__lc(const _Tp* __first1, const _Tp* __last1,
const _Up* __first2, const _Up* __last2)
{ return __3way(__first1, __last1, __first2, __last2) < 0; }
template<typename _Tp, typename _Up>
static ptrdiff_t
__3way(const _Tp* __first1, const _Tp* __last1,
const _Up* __first2, const _Up* __last2)
{
const size_t __len1 = __last1 - __first1;
const size_t __len2 = __last2 - __first2;
if (const size_t __len = std::min(__len1, __len2))
if (int __result = std::__memcmp(__first1, __first2, __len))
return __result;
return ptrdiff_t(__len1 - __len2);
}
};
template<typename _II1, typename _II2>
inline bool
__lexicographical_compare_aux1(_II1 __first1, _II1 __last1,
_II2 __first2, _II2 __last2)
{
typedef typename iterator_traits<_II1>::value_type _ValueType1;
typedef typename iterator_traits<_II2>::value_type _ValueType2;
const bool __simple =
(__is_memcmp_ordered_with<_ValueType1, _ValueType2>::__value
&& __is_pointer<_II1>::__value
&& __is_pointer<_II2>::__value
);
return std::__lexicographical_compare<__simple>::__lc(__first1, __last1,
__first2, __last2);
}
template<typename _Tp1, typename _Ref1, typename _Ptr1,
typename _Tp2>
bool
__lexicographical_compare_aux1(
std::_Deque_iterator<_Tp1, _Ref1, _Ptr1>,
std::_Deque_iterator<_Tp1, _Ref1, _Ptr1>,
_Tp2*, _Tp2*);
template<typename _Tp1,
typename _Tp2, typename _Ref2, typename _Ptr2>
bool
__lexicographical_compare_aux1(_Tp1*, _Tp1*,
std::_Deque_iterator<_Tp2, _Ref2, _Ptr2>,
std::_Deque_iterator<_Tp2, _Ref2, _Ptr2>);
template<typename _Tp1, typename _Ref1, typename _Ptr1,
typename _Tp2, typename _Ref2, typename _Ptr2>
bool
__lexicographical_compare_aux1(
std::_Deque_iterator<_Tp1, _Ref1, _Ptr1>,
std::_Deque_iterator<_Tp1, _Ref1, _Ptr1>,
std::_Deque_iterator<_Tp2, _Ref2, _Ptr2>,
std::_Deque_iterator<_Tp2, _Ref2, _Ptr2>);
template<typename _II1, typename _II2>
inline bool
__lexicographical_compare_aux(_II1 __first1, _II1 __last1,
_II2 __first2, _II2 __last2)
{
return std::__lexicographical_compare_aux1(std::__niter_base(__first1),
std::__niter_base(__last1),
std::__niter_base(__first2),
std::__niter_base(__last2));
}
template<typename _Iter1, typename _Seq1, typename _Cat1,
typename _II2>
bool
__lexicographical_compare_aux(
const ::__gnu_debug::_Safe_iterator<_Iter1, _Seq1, _Cat1>&,
const ::__gnu_debug::_Safe_iterator<_Iter1, _Seq1, _Cat1>&,
_II2, _II2);
template<typename _II1,
typename _Iter2, typename _Seq2, typename _Cat2>
bool
__lexicographical_compare_aux(
_II1, _II1,
const ::__gnu_debug::_Safe_iterator<_Iter2, _Seq2, _Cat2>&,
const ::__gnu_debug::_Safe_iterator<_Iter2, _Seq2, _Cat2>&);
template<typename _Iter1, typename _Seq1, typename _Cat1,
typename _Iter2, typename _Seq2, typename _Cat2>
bool
__lexicographical_compare_aux(
const ::__gnu_debug::_Safe_iterator<_Iter1, _Seq1, _Cat1>&,
const ::__gnu_debug::_Safe_iterator<_Iter1, _Seq1, _Cat1>&,
const ::__gnu_debug::_Safe_iterator<_Iter2, _Seq2, _Cat2>&,
const ::__gnu_debug::_Safe_iterator<_Iter2, _Seq2, _Cat2>&);
template<typename _ForwardIterator, typename _Tp, typename _Compare>
_ForwardIterator
__lower_bound(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __val, _Compare __comp)
{
typedef typename iterator_traits<_ForwardIterator>::difference_type
_DistanceType;
_DistanceType __len = std::distance(__first, __last);
while (__len > 0)
{
_DistanceType __half = __len >> 1;
_ForwardIterator __middle = __first;
std::advance(__middle, __half);
if (__comp(__middle, __val))
{
__first = __middle;
++__first;
__len = __len - __half - 1;
}
else
__len = __half;
}
return __first;
}
template<typename _ForwardIterator, typename _Tp>
inline _ForwardIterator
lower_bound(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __val)
{
;
return std::__lower_bound(__first, __last, __val,
__gnu_cxx::__ops::__iter_less_val());
}
template<typename _Tp>
inline constexpr _Tp
__lg(_Tp __n)
{
return std::__bit_width(make_unsigned_t<_Tp>(__n)) - 1;
}
template<typename _II1, typename _II2>
inline bool
equal(_II1 __first1, _II1 __last1, _II2 __first2)
{
;
return std::__equal_aux(__first1, __last1, __first2);
}
template<typename _IIter1, typename _IIter2, typename _BinaryPredicate>
inline bool
equal(_IIter1 __first1, _IIter1 __last1,
_IIter2 __first2, _BinaryPredicate __binary_pred)
{
;
for (; __first1 != __last1; ++__first1, (void)++__first2)
if (!bool(__binary_pred(*__first1, *__first2)))
return false;
return true;
}
template<typename _II1, typename _II2>
inline bool
__equal4(_II1 __first1, _II1 __last1, _II2 __first2, _II2 __last2)
{
using _RATag = random_access_iterator_tag;
using _Cat1 = typename iterator_traits<_II1>::iterator_category;
using _Cat2 = typename iterator_traits<_II2>::iterator_category;
using _RAIters = __and_<is_same<_Cat1, _RATag>, is_same<_Cat2, _RATag>>;
if (_RAIters())
{
auto __d1 = std::distance(__first1, __last1);
auto __d2 = std::distance(__first2, __last2);
if (__d1 != __d2)
return false;
return std::equal(__first1, __last1, __first2);
}
for (; __first1 != __last1 && __first2 != __last2;
++__first1, (void)++__first2)
if (!(*__first1 == *__first2))
return false;
return __first1 == __last1 && __first2 == __last2;
}
template<typename _II1, typename _II2, typename _BinaryPredicate>
inline bool
__equal4(_II1 __first1, _II1 __last1, _II2 __first2, _II2 __last2,
_BinaryPredicate __binary_pred)
{
using _RATag = random_access_iterator_tag;
using _Cat1 = typename iterator_traits<_II1>::iterator_category;
using _Cat2 = typename iterator_traits<_II2>::iterator_category;
using _RAIters = __and_<is_same<_Cat1, _RATag>, is_same<_Cat2, _RATag>>;
if (_RAIters())
{
auto __d1 = std::distance(__first1, __last1);
auto __d2 = std::distance(__first2, __last2);
if (__d1 != __d2)
return false;
return std::equal(__first1, __last1, __first2,
__binary_pred);
}
for (; __first1 != __last1 && __first2 != __last2;
++__first1, (void)++__first2)
if (!bool(__binary_pred(*__first1, *__first2)))
return false;
return __first1 == __last1 && __first2 == __last2;
}
template<typename _II1, typename _II2>
inline bool
equal(_II1 __first1, _II1 __last1, _II2 __first2, _II2 __last2)
{
;
;
return std::__equal4(__first1, __last1, __first2, __last2);
}
template<typename _IIter1, typename _IIter2, typename _BinaryPredicate>
inline bool
equal(_IIter1 __first1, _IIter1 __last1,
_IIter2 __first2, _IIter2 __last2, _BinaryPredicate __binary_pred)
{
;
;
return std::__equal4(__first1, __last1, __first2, __last2,
__binary_pred);
}
template<typename _II1, typename _II2>
inline bool
lexicographical_compare(_II1 __first1, _II1 __last1,
_II2 __first2, _II2 __last2)
{
;
;
return std::__lexicographical_compare_aux(__first1, __last1,
__first2, __last2);
}
template<typename _II1, typename _II2, typename _Compare>
inline bool
lexicographical_compare(_II1 __first1, _II1 __last1,
_II2 __first2, _II2 __last2, _Compare __comp)
{
;
;
return std::__lexicographical_compare_impl
(__first1, __last1, __first2, __last2,
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _InputIterator1, typename _InputIterator2,
typename _BinaryPredicate>
pair<_InputIterator1, _InputIterator2>
__mismatch(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _BinaryPredicate __binary_pred)
{
while (__first1 != __last1 && __binary_pred(__first1, __first2))
{
++__first1;
++__first2;
}
return pair<_InputIterator1, _InputIterator2>(__first1, __first2);
}
template<typename _InputIterator1, typename _InputIterator2>
inline pair<_InputIterator1, _InputIterator2>
mismatch(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2)
{
;
return std::__mismatch(__first1, __last1, __first2,
__gnu_cxx::__ops::__iter_equal_to_iter());
}
template<typename _InputIterator1, typename _InputIterator2,
typename _BinaryPredicate>
inline pair<_InputIterator1, _InputIterator2>
mismatch(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _BinaryPredicate __binary_pred)
{
;
return std::__mismatch(__first1, __last1, __first2,
__gnu_cxx::__ops::__iter_comp_iter(__binary_pred));
}
template<typename _InputIterator1, typename _InputIterator2,
typename _BinaryPredicate>
pair<_InputIterator1, _InputIterator2>
__mismatch(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_BinaryPredicate __binary_pred)
{
while (__first1 != __last1 && __first2 != __last2
&& __binary_pred(__first1, __first2))
{
++__first1;
++__first2;
}
return pair<_InputIterator1, _InputIterator2>(__first1, __first2);
}
template<typename _InputIterator1, typename _InputIterator2>
inline pair<_InputIterator1, _InputIterator2>
mismatch(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2)
{
;
;
return std::__mismatch(__first1, __last1, __first2, __last2,
__gnu_cxx::__ops::__iter_equal_to_iter());
}
template<typename _InputIterator1, typename _InputIterator2,
typename _BinaryPredicate>
inline pair<_InputIterator1, _InputIterator2>
mismatch(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_BinaryPredicate __binary_pred)
{
;
;
return std::__mismatch(__first1, __last1, __first2, __last2,
__gnu_cxx::__ops::__iter_comp_iter(__binary_pred));
}
template<typename _InputIterator, typename _Predicate>
inline _InputIterator
__find_if(_InputIterator __first, _InputIterator __last,
_Predicate __pred, input_iterator_tag)
{
while (__first != __last && !__pred(__first))
++__first;
return __first;
}
template<typename _RandomAccessIterator, typename _Predicate>
_RandomAccessIterator
__find_if(_RandomAccessIterator __first, _RandomAccessIterator __last,
_Predicate __pred, random_access_iterator_tag)
{
typename iterator_traits<_RandomAccessIterator>::difference_type
__trip_count = (__last - __first) >> 2;
for (; __trip_count > 0; --__trip_count)
{
if (__pred(__first))
return __first;
++__first;
if (__pred(__first))
return __first;
++__first;
if (__pred(__first))
return __first;
++__first;
if (__pred(__first))
return __first;
++__first;
}
switch (__last - __first)
{
case 3:
if (__pred(__first))
return __first;
++__first;
case 2:
if (__pred(__first))
return __first;
++__first;
case 1:
if (__pred(__first))
return __first;
++__first;
case 0:
default:
return __last;
}
}
template<typename _Iterator, typename _Predicate>
inline _Iterator
__find_if(_Iterator __first, _Iterator __last, _Predicate __pred)
{
return __find_if(__first, __last, __pred,
std::__iterator_category(__first));
}
template<typename _InputIterator, typename _Predicate>
typename iterator_traits<_InputIterator>::difference_type
__count_if(_InputIterator __first, _InputIterator __last, _Predicate __pred)
{
typename iterator_traits<_InputIterator>::difference_type __n = 0;
for (; __first != __last; ++__first)
if (__pred(__first))
++__n;
return __n;
}
template<typename _ForwardIterator, typename _Predicate>
_ForwardIterator
__remove_if(_ForwardIterator __first, _ForwardIterator __last,
_Predicate __pred)
{
__first = std::__find_if(__first, __last, __pred);
if (__first == __last)
return __first;
_ForwardIterator __result = __first;
++__first;
for (; __first != __last; ++__first)
if (!__pred(__first))
{
*__result = std::move(*__first);
++__result;
}
return __result;
}
template<typename _ForwardIterator1, typename _ForwardIterator2,
typename _BinaryPredicate>
bool
__is_permutation(_ForwardIterator1 __first1, _ForwardIterator1 __last1,
_ForwardIterator2 __first2, _BinaryPredicate __pred)
{
for (; __first1 != __last1; ++__first1, (void)++__first2)
if (!__pred(__first1, __first2))
break;
if (__first1 == __last1)
return true;
_ForwardIterator2 __last2 = __first2;
std::advance(__last2, std::distance(__first1, __last1));
for (_ForwardIterator1 __scan = __first1; __scan != __last1; ++__scan)
{
if (__scan != std::__find_if(__first1, __scan,
__gnu_cxx::__ops::__iter_comp_iter(__pred, __scan)))
continue;
auto __matches
= std::__count_if(__first2, __last2,
__gnu_cxx::__ops::__iter_comp_iter(__pred, __scan));
if (0 == __matches ||
std::__count_if(__scan, __last1,
__gnu_cxx::__ops::__iter_comp_iter(__pred, __scan))
!= __matches)
return false;
}
return true;
}
template<typename _ForwardIterator1, typename _ForwardIterator2>
inline bool
is_permutation(_ForwardIterator1 __first1, _ForwardIterator1 __last1,
_ForwardIterator2 __first2)
{
;
return std::__is_permutation(__first1, __last1, __first2,
__gnu_cxx::__ops::__iter_equal_to_iter());
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
enum float_round_style
{
round_indeterminate = -1,
round_toward_zero = 0,
round_to_nearest = 1,
round_toward_infinity = 2,
round_toward_neg_infinity = 3
};
enum float_denorm_style
{
denorm_indeterminate = -1,
denorm_absent = 0,
denorm_present = 1
};
struct __numeric_limits_base
{
static constexpr bool is_specialized = false;
static constexpr int digits = 0;
static constexpr int digits10 = 0;
static constexpr int max_digits10 = 0;
static constexpr bool is_signed = false;
static constexpr bool is_integer = false;
static constexpr bool is_exact = false;
static constexpr int radix = 0;
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr float_denorm_style has_denorm = denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = false;
static constexpr bool is_modulo = false;
static constexpr bool traps = false;
static constexpr bool tinyness_before = false;
static constexpr float_round_style round_style =
round_toward_zero;
};
template<typename _Tp>
struct numeric_limits : public __numeric_limits_base
{
static constexpr _Tp
min() noexcept { return _Tp(); }
static constexpr _Tp
max() noexcept { return _Tp(); }
static constexpr _Tp
lowest() noexcept { return _Tp(); }
static constexpr _Tp
epsilon() noexcept { return _Tp(); }
static constexpr _Tp
round_error() noexcept { return _Tp(); }
static constexpr _Tp
infinity() noexcept { return _Tp(); }
static constexpr _Tp
quiet_NaN() noexcept { return _Tp(); }
static constexpr _Tp
signaling_NaN() noexcept { return _Tp(); }
static constexpr _Tp
denorm_min() noexcept { return _Tp(); }
};
template<typename _Tp>
struct numeric_limits<const _Tp>
: public numeric_limits<_Tp> { };
template<typename _Tp>
struct numeric_limits<volatile _Tp>
: public numeric_limits<_Tp> { };
template<typename _Tp>
struct numeric_limits<const volatile _Tp>
: public numeric_limits<_Tp> { };
template<>
struct numeric_limits<bool>
{
static constexpr bool is_specialized = true;
static constexpr bool
min() noexcept { return false; }
static constexpr bool
max() noexcept { return true; }
static constexpr bool
lowest() noexcept { return min(); }
static constexpr int digits = 1;
static constexpr int digits10 = 0;
static constexpr int max_digits10 = 0;
static constexpr bool is_signed = false;
static constexpr bool is_integer = true;
static constexpr bool is_exact = true;
static constexpr int radix = 2;
static constexpr bool
epsilon() noexcept { return false; }
static constexpr bool
round_error() noexcept { return false; }
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr float_denorm_style has_denorm
= denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr bool
infinity() noexcept { return false; }
static constexpr bool
quiet_NaN() noexcept { return false; }
static constexpr bool
signaling_NaN() noexcept { return false; }
static constexpr bool
denorm_min() noexcept { return false; }
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = false;
static constexpr bool traps = true;
static constexpr bool tinyness_before = false;
static constexpr float_round_style round_style
= round_toward_zero;
};
template<>
struct numeric_limits<char>
{
static constexpr bool is_specialized = true;
static constexpr char
min() noexcept { return (((char)(-1) < 0) ? -(((char)(-1) < 0) ? (((((char)1 << ((sizeof(char) * 8 - ((char)(-1) < 0)) - 1)) - 1) << 1) + 1) : ~(char)0) - 1 : (char)0); }
static constexpr char
max() noexcept { return (((char)(-1) < 0) ? (((((char)1 << ((sizeof(char) * 8 - ((char)(-1) < 0)) - 1)) - 1) << 1) + 1) : ~(char)0); }
static constexpr char
lowest() noexcept { return min(); }
static constexpr int digits = (sizeof(char) * 8 - ((char)(-1) < 0));
static constexpr int digits10 = ((sizeof(char) * 8 - ((char)(-1) < 0)) * 643L / 2136);
static constexpr int max_digits10 = 0;
static constexpr bool is_signed = ((char)(-1) < 0);
static constexpr bool is_integer = true;
static constexpr bool is_exact = true;
static constexpr int radix = 2;
static constexpr char
epsilon() noexcept { return 0; }
static constexpr char
round_error() noexcept { return 0; }
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr float_denorm_style has_denorm
= denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr
char infinity() noexcept { return char(); }
static constexpr char
quiet_NaN() noexcept { return char(); }
static constexpr char
signaling_NaN() noexcept { return char(); }
static constexpr char
denorm_min() noexcept { return static_cast<char>(0); }
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = !is_signed;
static constexpr bool traps = true;
static constexpr bool tinyness_before = false;
static constexpr float_round_style round_style
= round_toward_zero;
};
template<>
struct numeric_limits<signed char>
{
static constexpr bool is_specialized = true;
static constexpr signed char
min() noexcept { return -0x7f - 1; }
static constexpr signed char
max() noexcept { return 0x7f; }
static constexpr signed char
lowest() noexcept { return min(); }
static constexpr int digits = (sizeof(signed char) * 8 - ((signed char)(-1) < 0));
static constexpr int digits10
= ((sizeof(signed char) * 8 - ((signed char)(-1) < 0)) * 643L / 2136);
static constexpr int max_digits10 = 0;
static constexpr bool is_signed = true;
static constexpr bool is_integer = true;
static constexpr bool is_exact = true;
static constexpr int radix = 2;
static constexpr signed char
epsilon() noexcept { return 0; }
static constexpr signed char
round_error() noexcept { return 0; }
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr float_denorm_style has_denorm
= denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr signed char
infinity() noexcept { return static_cast<signed char>(0); }
static constexpr signed char
quiet_NaN() noexcept { return static_cast<signed char>(0); }
static constexpr signed char
signaling_NaN() noexcept
{ return static_cast<signed char>(0); }
static constexpr signed char
denorm_min() noexcept
{ return static_cast<signed char>(0); }
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = false;
static constexpr bool traps = true;
static constexpr bool tinyness_before = false;
static constexpr float_round_style round_style
= round_toward_zero;
};
template<>
struct numeric_limits<unsigned char>
{
static constexpr bool is_specialized = true;
static constexpr unsigned char
min() noexcept { return 0; }
static constexpr unsigned char
max() noexcept { return 0x7f * 2U + 1; }
static constexpr unsigned char
lowest() noexcept { return min(); }
static constexpr int digits
= (sizeof(unsigned char) * 8 - ((unsigned char)(-1) < 0));
static constexpr int digits10
= ((sizeof(unsigned char) * 8 - ((unsigned char)(-1) < 0)) * 643L / 2136);
static constexpr int max_digits10 = 0;
static constexpr bool is_signed = false;
static constexpr bool is_integer = true;
static constexpr bool is_exact = true;
static constexpr int radix = 2;
static constexpr unsigned char
epsilon() noexcept { return 0; }
static constexpr unsigned char
round_error() noexcept { return 0; }
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr float_denorm_style has_denorm
= denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr unsigned char
infinity() noexcept
{ return static_cast<unsigned char>(0); }
static constexpr unsigned char
quiet_NaN() noexcept
{ return static_cast<unsigned char>(0); }
static constexpr unsigned char
signaling_NaN() noexcept
{ return static_cast<unsigned char>(0); }
static constexpr unsigned char
denorm_min() noexcept
{ return static_cast<unsigned char>(0); }
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = true;
static constexpr bool traps = true;
static constexpr bool tinyness_before = false;
static constexpr float_round_style round_style
= round_toward_zero;
};
template<>
struct numeric_limits<wchar_t>
{
static constexpr bool is_specialized = true;
static constexpr wchar_t
min() noexcept { return (((wchar_t)(-1) < 0) ? -(((wchar_t)(-1) < 0) ? (((((wchar_t)1 << ((sizeof(wchar_t) * 8 - ((wchar_t)(-1) < 0)) - 1)) - 1) << 1) + 1) : ~(wchar_t)0) - 1 : (wchar_t)0); }
static constexpr wchar_t
max() noexcept { return (((wchar_t)(-1) < 0) ? (((((wchar_t)1 << ((sizeof(wchar_t) * 8 - ((wchar_t)(-1) < 0)) - 1)) - 1) << 1) + 1) : ~(wchar_t)0); }
static constexpr wchar_t
lowest() noexcept { return min(); }
static constexpr int digits = (sizeof(wchar_t) * 8 - ((wchar_t)(-1) < 0));
static constexpr int digits10
= ((sizeof(wchar_t) * 8 - ((wchar_t)(-1) < 0)) * 643L / 2136);
static constexpr int max_digits10 = 0;
static constexpr bool is_signed = ((wchar_t)(-1) < 0);
static constexpr bool is_integer = true;
static constexpr bool is_exact = true;
static constexpr int radix = 2;
static constexpr wchar_t
epsilon() noexcept { return 0; }
static constexpr wchar_t
round_error() noexcept { return 0; }
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr float_denorm_style has_denorm
= denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr wchar_t
infinity() noexcept { return wchar_t(); }
static constexpr wchar_t
quiet_NaN() noexcept { return wchar_t(); }
static constexpr wchar_t
signaling_NaN() noexcept { return wchar_t(); }
static constexpr wchar_t
denorm_min() noexcept { return wchar_t(); }
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = !is_signed;
static constexpr bool traps = true;
static constexpr bool tinyness_before = false;
static constexpr float_round_style round_style
= round_toward_zero;
};
template<>
struct numeric_limits<char16_t>
{
static constexpr bool is_specialized = true;
static constexpr char16_t
min() noexcept { return (((char16_t)(-1) < 0) ? -(((char16_t)(-1) < 0) ? (((((char16_t)1 << ((sizeof(char16_t) * 8 - ((char16_t)(-1) < 0)) - 1)) - 1) << 1) + 1) : ~(char16_t)0) - 1 : (char16_t)0); }
static constexpr char16_t
max() noexcept { return (((char16_t)(-1) < 0) ? (((((char16_t)1 << ((sizeof(char16_t) * 8 - ((char16_t)(-1) < 0)) - 1)) - 1) << 1) + 1) : ~(char16_t)0); }
static constexpr char16_t
lowest() noexcept { return min(); }
static constexpr int digits = (sizeof(char16_t) * 8 - ((char16_t)(-1) < 0));
static constexpr int digits10 = ((sizeof(char16_t) * 8 - ((char16_t)(-1) < 0)) * 643L / 2136);
static constexpr int max_digits10 = 0;
static constexpr bool is_signed = ((char16_t)(-1) < 0);
static constexpr bool is_integer = true;
static constexpr bool is_exact = true;
static constexpr int radix = 2;
static constexpr char16_t
epsilon() noexcept { return 0; }
static constexpr char16_t
round_error() noexcept { return 0; }
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr float_denorm_style has_denorm = denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr char16_t
infinity() noexcept { return char16_t(); }
static constexpr char16_t
quiet_NaN() noexcept { return char16_t(); }
static constexpr char16_t
signaling_NaN() noexcept { return char16_t(); }
static constexpr char16_t
denorm_min() noexcept { return char16_t(); }
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = !is_signed;
static constexpr bool traps = true;
static constexpr bool tinyness_before = false;
static constexpr float_round_style round_style = round_toward_zero;
};
template<>
struct numeric_limits<char32_t>
{
static constexpr bool is_specialized = true;
static constexpr char32_t
min() noexcept { return (((char32_t)(-1) < 0) ? -(((char32_t)(-1) < 0) ? (((((char32_t)1 << ((sizeof(char32_t) * 8 - ((char32_t)(-1) < 0)) - 1)) - 1) << 1) + 1) : ~(char32_t)0) - 1 : (char32_t)0); }
static constexpr char32_t
max() noexcept { return (((char32_t)(-1) < 0) ? (((((char32_t)1 << ((sizeof(char32_t) * 8 - ((char32_t)(-1) < 0)) - 1)) - 1) << 1) + 1) : ~(char32_t)0); }
static constexpr char32_t
lowest() noexcept { return min(); }
static constexpr int digits = (sizeof(char32_t) * 8 - ((char32_t)(-1) < 0));
static constexpr int digits10 = ((sizeof(char32_t) * 8 - ((char32_t)(-1) < 0)) * 643L / 2136);
static constexpr int max_digits10 = 0;
static constexpr bool is_signed = ((char32_t)(-1) < 0);
static constexpr bool is_integer = true;
static constexpr bool is_exact = true;
static constexpr int radix = 2;
static constexpr char32_t
epsilon() noexcept { return 0; }
static constexpr char32_t
round_error() noexcept { return 0; }
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr float_denorm_style has_denorm = denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr char32_t
infinity() noexcept { return char32_t(); }
static constexpr char32_t
quiet_NaN() noexcept { return char32_t(); }
static constexpr char32_t
signaling_NaN() noexcept { return char32_t(); }
static constexpr char32_t
denorm_min() noexcept { return char32_t(); }
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = !is_signed;
static constexpr bool traps = true;
static constexpr bool tinyness_before = false;
static constexpr float_round_style round_style = round_toward_zero;
};
template<>
struct numeric_limits<short>
{
static constexpr bool is_specialized = true;
static constexpr short
min() noexcept { return -0x7fff - 1; }
static constexpr short
max() noexcept { return 0x7fff; }
static constexpr short
lowest() noexcept { return min(); }
static constexpr int digits = (sizeof(short) * 8 - ((short)(-1) < 0));
static constexpr int digits10 = ((sizeof(short) * 8 - ((short)(-1) < 0)) * 643L / 2136);
static constexpr int max_digits10 = 0;
static constexpr bool is_signed = true;
static constexpr bool is_integer = true;
static constexpr bool is_exact = true;
static constexpr int radix = 2;
static constexpr short
epsilon() noexcept { return 0; }
static constexpr short
round_error() noexcept { return 0; }
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr float_denorm_style has_denorm
= denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr short
infinity() noexcept { return short(); }
static constexpr short
quiet_NaN() noexcept { return short(); }
static constexpr short
signaling_NaN() noexcept { return short(); }
static constexpr short
denorm_min() noexcept { return short(); }
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = false;
static constexpr bool traps = true;
static constexpr bool tinyness_before = false;
static constexpr float_round_style round_style
= round_toward_zero;
};
template<>
struct numeric_limits<unsigned short>
{
static constexpr bool is_specialized = true;
static constexpr unsigned short
min() noexcept { return 0; }
static constexpr unsigned short
max() noexcept { return 0x7fff * 2U + 1; }
static constexpr unsigned short
lowest() noexcept { return min(); }
static constexpr int digits
= (sizeof(unsigned short) * 8 - ((unsigned short)(-1) < 0));
static constexpr int digits10
= ((sizeof(unsigned short) * 8 - ((unsigned short)(-1) < 0)) * 643L / 2136);
static constexpr int max_digits10 = 0;
static constexpr bool is_signed = false;
static constexpr bool is_integer = true;
static constexpr bool is_exact = true;
static constexpr int radix = 2;
static constexpr unsigned short
epsilon() noexcept { return 0; }
static constexpr unsigned short
round_error() noexcept { return 0; }
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr float_denorm_style has_denorm
= denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr unsigned short
infinity() noexcept
{ return static_cast<unsigned short>(0); }
static constexpr unsigned short
quiet_NaN() noexcept
{ return static_cast<unsigned short>(0); }
static constexpr unsigned short
signaling_NaN() noexcept
{ return static_cast<unsigned short>(0); }
static constexpr unsigned short
denorm_min() noexcept
{ return static_cast<unsigned short>(0); }
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = true;
static constexpr bool traps = true;
static constexpr bool tinyness_before = false;
static constexpr float_round_style round_style
= round_toward_zero;
};
template<>
struct numeric_limits<int>
{
static constexpr bool is_specialized = true;
static constexpr int
min() noexcept { return -0x7fffffff - 1; }
static constexpr int
max() noexcept { return 0x7fffffff; }
static constexpr int
lowest() noexcept { return min(); }
static constexpr int digits = (sizeof(int) * 8 - ((int)(-1) < 0));
static constexpr int digits10 = ((sizeof(int) * 8 - ((int)(-1) < 0)) * 643L / 2136);
static constexpr int max_digits10 = 0;
static constexpr bool is_signed = true;
static constexpr bool is_integer = true;
static constexpr bool is_exact = true;
static constexpr int radix = 2;
static constexpr int
epsilon() noexcept { return 0; }
static constexpr int
round_error() noexcept { return 0; }
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr float_denorm_style has_denorm
= denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr int
infinity() noexcept { return static_cast<int>(0); }
static constexpr int
quiet_NaN() noexcept { return static_cast<int>(0); }
static constexpr int
signaling_NaN() noexcept { return static_cast<int>(0); }
static constexpr int
denorm_min() noexcept { return static_cast<int>(0); }
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = false;
static constexpr bool traps = true;
static constexpr bool tinyness_before = false;
static constexpr float_round_style round_style
= round_toward_zero;
};
template<>
struct numeric_limits<unsigned int>
{
static constexpr bool is_specialized = true;
static constexpr unsigned int
min() noexcept { return 0; }
static constexpr unsigned int
max() noexcept { return 0x7fffffff * 2U + 1; }
static constexpr unsigned int
lowest() noexcept { return min(); }
static constexpr int digits
= (sizeof(unsigned int) * 8 - ((unsigned int)(-1) < 0));
static constexpr int digits10
= ((sizeof(unsigned int) * 8 - ((unsigned int)(-1) < 0)) * 643L / 2136);
static constexpr int max_digits10 = 0;
static constexpr bool is_signed = false;
static constexpr bool is_integer = true;
static constexpr bool is_exact = true;
static constexpr int radix = 2;
static constexpr unsigned int
epsilon() noexcept { return 0; }
static constexpr unsigned int
round_error() noexcept { return 0; }
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr float_denorm_style has_denorm
= denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr unsigned int
infinity() noexcept { return static_cast<unsigned int>(0); }
static constexpr unsigned int
quiet_NaN() noexcept
{ return static_cast<unsigned int>(0); }
static constexpr unsigned int
signaling_NaN() noexcept
{ return static_cast<unsigned int>(0); }
static constexpr unsigned int
denorm_min() noexcept
{ return static_cast<unsigned int>(0); }
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = true;
static constexpr bool traps = true;
static constexpr bool tinyness_before = false;
static constexpr float_round_style round_style
= round_toward_zero;
};
template<>
struct numeric_limits<long>
{
static constexpr bool is_specialized = true;
static constexpr long
min() noexcept { return -0x7fffffffL - 1; }
static constexpr long
max() noexcept { return 0x7fffffffL; }
static constexpr long
lowest() noexcept { return min(); }
static constexpr int digits = (sizeof(long) * 8 - ((long)(-1) < 0));
static constexpr int digits10 = ((sizeof(long) * 8 - ((long)(-1) < 0)) * 643L / 2136);
static constexpr int max_digits10 = 0;
static constexpr bool is_signed = true;
static constexpr bool is_integer = true;
static constexpr bool is_exact = true;
static constexpr int radix = 2;
static constexpr long
epsilon() noexcept { return 0; }
static constexpr long
round_error() noexcept { return 0; }
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr float_denorm_style has_denorm
= denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr long
infinity() noexcept { return static_cast<long>(0); }
static constexpr long
quiet_NaN() noexcept { return static_cast<long>(0); }
static constexpr long
signaling_NaN() noexcept { return static_cast<long>(0); }
static constexpr long
denorm_min() noexcept { return static_cast<long>(0); }
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = false;
static constexpr bool traps = true;
static constexpr bool tinyness_before = false;
static constexpr float_round_style round_style
= round_toward_zero;
};
template<>
struct numeric_limits<unsigned long>
{
static constexpr bool is_specialized = true;
static constexpr unsigned long
min() noexcept { return 0; }
static constexpr unsigned long
max() noexcept { return 0x7fffffffL * 2UL + 1; }
static constexpr unsigned long
lowest() noexcept { return min(); }
static constexpr int digits
= (sizeof(unsigned long) * 8 - ((unsigned long)(-1) < 0));
static constexpr int digits10
= ((sizeof(unsigned long) * 8 - ((unsigned long)(-1) < 0)) * 643L / 2136);
static constexpr int max_digits10 = 0;
static constexpr bool is_signed = false;
static constexpr bool is_integer = true;
static constexpr bool is_exact = true;
static constexpr int radix = 2;
static constexpr unsigned long
epsilon() noexcept { return 0; }
static constexpr unsigned long
round_error() noexcept { return 0; }
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr float_denorm_style has_denorm
= denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr unsigned long
infinity() noexcept
{ return static_cast<unsigned long>(0); }
static constexpr unsigned long
quiet_NaN() noexcept
{ return static_cast<unsigned long>(0); }
static constexpr unsigned long
signaling_NaN() noexcept
{ return static_cast<unsigned long>(0); }
static constexpr unsigned long
denorm_min() noexcept
{ return static_cast<unsigned long>(0); }
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = true;
static constexpr bool traps = true;
static constexpr bool tinyness_before = false;
static constexpr float_round_style round_style
= round_toward_zero;
};
template<>
struct numeric_limits<long long>
{
static constexpr bool is_specialized = true;
static constexpr long long
min() noexcept { return -0x7fffffffffffffffLL - 1; }
static constexpr long long
max() noexcept { return 0x7fffffffffffffffLL; }
static constexpr long long
lowest() noexcept { return min(); }
static constexpr int digits
= (sizeof(long long) * 8 - ((long long)(-1) < 0));
static constexpr int digits10
= ((sizeof(long long) * 8 - ((long long)(-1) < 0)) * 643L / 2136);
static constexpr int max_digits10 = 0;
static constexpr bool is_signed = true;
static constexpr bool is_integer = true;
static constexpr bool is_exact = true;
static constexpr int radix = 2;
static constexpr long long
epsilon() noexcept { return 0; }
static constexpr long long
round_error() noexcept { return 0; }
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr float_denorm_style has_denorm
= denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr long long
infinity() noexcept { return static_cast<long long>(0); }
static constexpr long long
quiet_NaN() noexcept { return static_cast<long long>(0); }
static constexpr long long
signaling_NaN() noexcept
{ return static_cast<long long>(0); }
static constexpr long long
denorm_min() noexcept { return static_cast<long long>(0); }
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = false;
static constexpr bool traps = true;
static constexpr bool tinyness_before = false;
static constexpr float_round_style round_style
= round_toward_zero;
};
template<>
struct numeric_limits<unsigned long long>
{
static constexpr bool is_specialized = true;
static constexpr unsigned long long
min() noexcept { return 0; }
static constexpr unsigned long long
max() noexcept { return 0x7fffffffffffffffLL * 2ULL + 1; }
static constexpr unsigned long long
lowest() noexcept { return min(); }
static constexpr int digits
= (sizeof(unsigned long long) * 8 - ((unsigned long long)(-1) < 0));
static constexpr int digits10
= ((sizeof(unsigned long long) * 8 - ((unsigned long long)(-1) < 0)) * 643L / 2136);
static constexpr int max_digits10 = 0;
static constexpr bool is_signed = false;
static constexpr bool is_integer = true;
static constexpr bool is_exact = true;
static constexpr int radix = 2;
static constexpr unsigned long long
epsilon() noexcept { return 0; }
static constexpr unsigned long long
round_error() noexcept { return 0; }
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr float_denorm_style has_denorm
= denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr unsigned long long
infinity() noexcept
{ return static_cast<unsigned long long>(0); }
static constexpr unsigned long long
quiet_NaN() noexcept
{ return static_cast<unsigned long long>(0); }
static constexpr unsigned long long
signaling_NaN() noexcept
{ return static_cast<unsigned long long>(0); }
static constexpr unsigned long long
denorm_min() noexcept
{ return static_cast<unsigned long long>(0); }
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = true;
static constexpr bool traps = true;
static constexpr bool tinyness_before = false;
static constexpr float_round_style round_style
= round_toward_zero;
};
template<>
struct numeric_limits<float>
{
static constexpr bool is_specialized = true;
static constexpr float
min() noexcept { return 1.17549435082228750796873653722224568e-38F; }
static constexpr float
max() noexcept { return 3.40282346638528859811704183484516925e+38F; }
static constexpr float
lowest() noexcept { return -3.40282346638528859811704183484516925e+38F; }
static constexpr int digits = 24;
static constexpr int digits10 = 6;
static constexpr int max_digits10
= (2 + (24) * 643L / 2136);
static constexpr bool is_signed = true;
static constexpr bool is_integer = false;
static constexpr bool is_exact = false;
static constexpr int radix = 2;
static constexpr float
epsilon() noexcept { return 1.19209289550781250000000000000000000e-7F; }
static constexpr float
round_error() noexcept { return 0.5F; }
static constexpr int min_exponent = (-125);
static constexpr int min_exponent10 = (-37);
static constexpr int max_exponent = 128;
static constexpr int max_exponent10 = 38;
static constexpr bool has_infinity = 1;
static constexpr bool has_quiet_NaN = 1;
static constexpr bool has_signaling_NaN = has_quiet_NaN;
static constexpr float_denorm_style has_denorm
= bool(1) ? denorm_present : denorm_absent;
static constexpr bool has_denorm_loss
= false;
static constexpr float
infinity() noexcept { return __builtin_huge_valf(); }
static constexpr float
quiet_NaN() noexcept { return __builtin_nanf(""); }
static constexpr float
signaling_NaN() noexcept { return __builtin_nansf(""); }
static constexpr float
denorm_min() noexcept { return 1.40129846432481707092372958328991613e-45F; }
static constexpr bool is_iec559
= has_infinity && has_quiet_NaN && has_denorm == denorm_present;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = false;
static constexpr bool traps = false;
static constexpr bool tinyness_before
= false;
static constexpr float_round_style round_style
= round_to_nearest;
};
template<>
struct numeric_limits<double>
{
static constexpr bool is_specialized = true;
static constexpr double
min() noexcept { return double(2.22507385850720138309023271733240406e-308L); }
static constexpr double
max() noexcept { return double(1.79769313486231570814527423731704357e+308L); }
static constexpr double
lowest() noexcept { return -double(1.79769313486231570814527423731704357e+308L); }
static constexpr int digits = 53;
static constexpr int digits10 = 15;
static constexpr int max_digits10
= (2 + (53) * 643L / 2136);
static constexpr bool is_signed = true;
static constexpr bool is_integer = false;
static constexpr bool is_exact = false;
static constexpr int radix = 2;
static constexpr double
epsilon() noexcept { return double(2.22044604925031308084726333618164062e-16L); }
static constexpr double
round_error() noexcept { return 0.5; }
static constexpr int min_exponent = (-1021);
static constexpr int min_exponent10 = (-307);
static constexpr int max_exponent = 1024;
static constexpr int max_exponent10 = 308;
static constexpr bool has_infinity = 1;
static constexpr bool has_quiet_NaN = 1;
static constexpr bool has_signaling_NaN = has_quiet_NaN;
static constexpr float_denorm_style has_denorm
= bool(1) ? denorm_present : denorm_absent;
static constexpr bool has_denorm_loss
= false;
static constexpr double
infinity() noexcept { return __builtin_huge_val(); }
static constexpr double
quiet_NaN() noexcept { return __builtin_nan(""); }
static constexpr double
signaling_NaN() noexcept { return __builtin_nans(""); }
static constexpr double
denorm_min() noexcept { return double(4.94065645841246544176568792868221372e-324L); }
static constexpr bool is_iec559
= has_infinity && has_quiet_NaN && has_denorm == denorm_present;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = false;
static constexpr bool traps = false;
static constexpr bool tinyness_before
= false;
static constexpr float_round_style round_style
= round_to_nearest;
};
template<>
struct numeric_limits<long double>
{
static constexpr bool is_specialized = true;
static constexpr long double
min() noexcept { return 3.36210314311209350626267781732175260e-4932L; }
static constexpr long double
max() noexcept { return 1.18973149535723176502126385303097021e+4932L; }
static constexpr long double
lowest() noexcept { return -1.18973149535723176502126385303097021e+4932L; }
static constexpr int digits = 64;
static constexpr int digits10 = 18;
static constexpr int max_digits10
= (2 + (64) * 643L / 2136);
static constexpr bool is_signed = true;
static constexpr bool is_integer = false;
static constexpr bool is_exact = false;
static constexpr int radix = 2;
static constexpr long double
epsilon() noexcept { return 1.08420217248550443400745280086994171e-19L; }
static constexpr long double
round_error() noexcept { return 0.5L; }
static constexpr int min_exponent = (-16381);
static constexpr int min_exponent10 = (-4931);
static constexpr int max_exponent = 16384;
static constexpr int max_exponent10 = 4932;
static constexpr bool has_infinity = 1;
static constexpr bool has_quiet_NaN = 1;
static constexpr bool has_signaling_NaN = has_quiet_NaN;
static constexpr float_denorm_style has_denorm
= bool(1) ? denorm_present : denorm_absent;
static constexpr bool has_denorm_loss
= false;
static constexpr long double
infinity() noexcept { return __builtin_huge_vall(); }
static constexpr long double
quiet_NaN() noexcept { return __builtin_nanl(""); }
static constexpr long double
signaling_NaN() noexcept { return __builtin_nansl(""); }
static constexpr long double
denorm_min() noexcept { return 3.64519953188247460252840593361941982e-4951L; }
static constexpr bool is_iec559
= has_infinity && has_quiet_NaN && has_denorm == denorm_present;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = false;
static constexpr bool traps = false;
static constexpr bool tinyness_before =
false;
static constexpr float_round_style round_style =
round_to_nearest;
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __detail
{
template<typename _Tp>
struct __floating_point_constant
{
static const _Tp __value;
};
template<typename _Tp>
struct __numeric_constants
{
static _Tp __pi() throw()
{ return static_cast<_Tp>(3.1415926535897932384626433832795029L); }
static _Tp __pi_2() throw()
{ return static_cast<_Tp>(1.5707963267948966192313216916397514L); }
static _Tp __pi_3() throw()
{ return static_cast<_Tp>(1.0471975511965977461542144610931676L); }
static _Tp __pi_4() throw()
{ return static_cast<_Tp>(0.7853981633974483096156608458198757L); }
static _Tp __1_pi() throw()
{ return static_cast<_Tp>(0.3183098861837906715377675267450287L); }
static _Tp __2_sqrtpi() throw()
{ return static_cast<_Tp>(1.1283791670955125738961589031215452L); }
static _Tp __sqrt2() throw()
{ return static_cast<_Tp>(1.4142135623730950488016887242096981L); }
static _Tp __sqrt3() throw()
{ return static_cast<_Tp>(1.7320508075688772935274463415058723L); }
static _Tp __sqrtpio2() throw()
{ return static_cast<_Tp>(1.2533141373155002512078826424055226L); }
static _Tp __sqrt1_2() throw()
{ return static_cast<_Tp>(0.7071067811865475244008443621048490L); }
static _Tp __lnpi() throw()
{ return static_cast<_Tp>(1.1447298858494001741434273513530587L); }
static _Tp __gamma_e() throw()
{ return static_cast<_Tp>(0.5772156649015328606065120900824024L); }
static _Tp __euler() throw()
{ return static_cast<_Tp>(2.7182818284590452353602874713526625L); }
};
template<typename _Tp>
inline bool __isnan(_Tp __x)
{ return std::isnan(__x); }
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __detail
{
template <typename _Tp>
_Tp
__bernoulli_series(unsigned int __n)
{
static const _Tp __num[28] = {
_Tp(1UL), -_Tp(1UL) / _Tp(2UL),
_Tp(1UL) / _Tp(6UL), _Tp(0UL),
-_Tp(1UL) / _Tp(30UL), _Tp(0UL),
_Tp(1UL) / _Tp(42UL), _Tp(0UL),
-_Tp(1UL) / _Tp(30UL), _Tp(0UL),
_Tp(5UL) / _Tp(66UL), _Tp(0UL),
-_Tp(691UL) / _Tp(2730UL), _Tp(0UL),
_Tp(7UL) / _Tp(6UL), _Tp(0UL),
-_Tp(3617UL) / _Tp(510UL), _Tp(0UL),
_Tp(43867UL) / _Tp(798UL), _Tp(0UL),
-_Tp(174611) / _Tp(330UL), _Tp(0UL),
_Tp(854513UL) / _Tp(138UL), _Tp(0UL),
-_Tp(236364091UL) / _Tp(2730UL), _Tp(0UL),
_Tp(8553103UL) / _Tp(6UL), _Tp(0UL)
};
if (__n == 0)
return _Tp(1);
if (__n == 1)
return -_Tp(1) / _Tp(2);
if (__n % 2 == 1)
return _Tp(0);
if (__n < 28)
return __num[__n];
_Tp __fact = _Tp(1);
if ((__n / 2) % 2 == 0)
__fact *= _Tp(-1);
for (unsigned int __k = 1; __k <= __n; ++__k)
__fact *= __k / (_Tp(2) * __numeric_constants<_Tp>::__pi());
__fact *= _Tp(2);
_Tp __sum = _Tp(0);
for (unsigned int __i = 1; __i < 1000; ++__i)
{
_Tp __term = std::pow(_Tp(__i), -_Tp(__n));
if (__term < std::numeric_limits<_Tp>::epsilon())
break;
__sum += __term;
}
return __fact * __sum;
}
template<typename _Tp>
inline _Tp
__bernoulli(int __n)
{ return __bernoulli_series<_Tp>(__n); }
template<typename _Tp>
_Tp
__log_gamma_bernoulli(_Tp __x)
{
_Tp __lg = (__x - _Tp(0.5L)) * std::log(__x) - __x
+ _Tp(0.5L) * std::log(_Tp(2)
* __numeric_constants<_Tp>::__pi());
const _Tp __xx = __x * __x;
_Tp __help = _Tp(1) / __x;
for ( unsigned int __i = 1; __i < 20; ++__i )
{
const _Tp __2i = _Tp(2 * __i);
__help /= __2i * (__2i - _Tp(1)) * __xx;
__lg += __bernoulli<_Tp>(2 * __i) * __help;
}
return __lg;
}
template<typename _Tp>
_Tp
__log_gamma_lanczos(_Tp __x)
{
const _Tp __xm1 = __x - _Tp(1);
static const _Tp __lanczos_cheb_7[9] = {
_Tp( 0.99999999999980993227684700473478L),
_Tp( 676.520368121885098567009190444019L),
_Tp(-1259.13921672240287047156078755283L),
_Tp( 771.3234287776530788486528258894L),
_Tp(-176.61502916214059906584551354L),
_Tp( 12.507343278686904814458936853L),
_Tp(-0.13857109526572011689554707L),
_Tp( 9.984369578019570859563e-6L),
_Tp( 1.50563273514931155834e-7L)
};
static const _Tp __LOGROOT2PI
= _Tp(0.9189385332046727417803297364056176L);
_Tp __sum = __lanczos_cheb_7[0];
for(unsigned int __k = 1; __k < 9; ++__k)
__sum += __lanczos_cheb_7[__k] / (__xm1 + __k);
const _Tp __term1 = (__xm1 + _Tp(0.5L))
* std::log((__xm1 + _Tp(7.5L))
/ __numeric_constants<_Tp>::__euler());
const _Tp __term2 = __LOGROOT2PI + std::log(__sum);
const _Tp __result = __term1 + (__term2 - _Tp(7));
return __result;
}
template<typename _Tp>
_Tp
__log_gamma(_Tp __x)
{
if (__x > _Tp(0.5L))
return __log_gamma_lanczos(__x);
else
{
const _Tp __sin_fact
= std::abs(std::sin(__numeric_constants<_Tp>::__pi() * __x));
if (__sin_fact == _Tp(0))
std::__throw_domain_error(("Argument is nonpositive integer " "in __log_gamma")
);
return __numeric_constants<_Tp>::__lnpi()
- std::log(__sin_fact)
- __log_gamma_lanczos(_Tp(1) - __x);
}
}
template<typename _Tp>
_Tp
__log_gamma_sign(_Tp __x)
{
if (__x > _Tp(0))
return _Tp(1);
else
{
const _Tp __sin_fact
= std::sin(__numeric_constants<_Tp>::__pi() * __x);
if (__sin_fact > _Tp(0))
return (1);
else if (__sin_fact < _Tp(0))
return -_Tp(1);
else
return _Tp(0);
}
}
template<typename _Tp>
_Tp
__log_bincoef(unsigned int __n, unsigned int __k)
{
static const _Tp __max_bincoeff
= std::numeric_limits<_Tp>::max_exponent10
* std::log(_Tp(10)) - _Tp(1);
_Tp __coeff = ::std::lgamma(_Tp(1 + __n))
- ::std::lgamma(_Tp(1 + __k))
- ::std::lgamma(_Tp(1 + __n - __k));
}
template<typename _Tp>
_Tp
__bincoef(unsigned int __n, unsigned int __k)
{
static const _Tp __max_bincoeff
= std::numeric_limits<_Tp>::max_exponent10
* std::log(_Tp(10)) - _Tp(1);
const _Tp __log_coeff = __log_bincoef<_Tp>(__n, __k);
if (__log_coeff > __max_bincoeff)
return std::numeric_limits<_Tp>::quiet_NaN();
else
return std::exp(__log_coeff);
}
template<typename _Tp>
inline _Tp
__gamma(_Tp __x)
{ return std::exp(__log_gamma(__x)); }
template<typename _Tp>
_Tp
__psi_series(_Tp __x)
{
_Tp __sum = -__numeric_constants<_Tp>::__gamma_e() - _Tp(1) / __x;
const unsigned int __max_iter = 100000;
for (unsigned int __k = 1; __k < __max_iter; ++__k)
{
const _Tp __term = __x / (__k * (__k + __x));
__sum += __term;
if (std::abs(__term / __sum) < std::numeric_limits<_Tp>::epsilon())
break;
}
return __sum;
}
template<typename _Tp>
_Tp
__psi_asymp(_Tp __x)
{
_Tp __sum = std::log(__x) - _Tp(0.5L) / __x;
const _Tp __xx = __x * __x;
_Tp __xp = __xx;
const unsigned int __max_iter = 100;
for (unsigned int __k = 1; __k < __max_iter; ++__k)
{
const _Tp __term = __bernoulli<_Tp>(2 * __k) / (2 * __k * __xp);
__sum -= __term;
if (std::abs(__term / __sum) < std::numeric_limits<_Tp>::epsilon())
break;
__xp *= __xx;
}
return __sum;
}
template<typename _Tp>
_Tp
__psi(_Tp __x)
{
const int __n = static_cast<int>(__x + 0.5L);
const _Tp __eps = _Tp(4) * std::numeric_limits<_Tp>::epsilon();
if (__n <= 0 && std::abs(__x - _Tp(__n)) < __eps)
return std::numeric_limits<_Tp>::quiet_NaN();
else if (__x < _Tp(0))
{
const _Tp __pi = __numeric_constants<_Tp>::__pi();
return __psi(_Tp(1) - __x)
- __pi * std::cos(__pi * __x) / std::sin(__pi * __x);
}
else if (__x > _Tp(100))
return __psi_asymp(__x);
else
return __psi_series(__x);
}
template<typename _Tp>
_Tp
__psi(unsigned int __n, _Tp __x)
{
if (__x <= _Tp(0))
std::__throw_domain_error(("Argument out of range " "in __psi")
);
else if (__n == 0)
return __psi(__x);
else
{
const _Tp __hzeta = __hurwitz_zeta(_Tp(__n + 1), __x);
const _Tp __ln_nfact = ::std::lgamma(_Tp(__n + 1));
_Tp __result = std::exp(__ln_nfact) * __hzeta;
if (__n % 2 == 1)
__result = -__result;
return __result;
}
}
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __detail
{
template <typename _Tp>
void
__gamma_temme(_Tp __mu,
_Tp & __gam1, _Tp & __gam2, _Tp & __gampl, _Tp & __gammi)
{
__gampl = _Tp(1) / ::std::tgamma(_Tp(1) + __mu);
__gammi = _Tp(1) / ::std::tgamma(_Tp(1) - __mu);
if (std::abs(__mu) < std::numeric_limits<_Tp>::epsilon())
__gam1 = -_Tp(__numeric_constants<_Tp>::__gamma_e());
else
__gam1 = (__gammi - __gampl) / (_Tp(2) * __mu);
__gam2 = (__gammi + __gampl) / (_Tp(2));
return;
}
template <typename _Tp>
void
__bessel_jn(_Tp __nu, _Tp __x,
_Tp & __Jnu, _Tp & __Nnu, _Tp & __Jpnu, _Tp & __Npnu)
{
if (__x == _Tp(0))
{
if (__nu == _Tp(0))
{
__Jnu = _Tp(1);
__Jpnu = _Tp(0);
}
else if (__nu == _Tp(1))
{
__Jnu = _Tp(0);
__Jpnu = _Tp(0.5L);
}
else
{
__Jnu = _Tp(0);
__Jpnu = _Tp(0);
}
__Nnu = -std::numeric_limits<_Tp>::infinity();
__Npnu = std::numeric_limits<_Tp>::infinity();
return;
}
const _Tp __eps = std::numeric_limits<_Tp>::epsilon();
const _Tp __fp_min = std::sqrt(std::numeric_limits<_Tp>::min());
const int __max_iter = 15000;
const _Tp __x_min = _Tp(2);
const int __nl = (__x < __x_min
? static_cast<int>(__nu + _Tp(0.5L))
: std::max(0, static_cast<int>(__nu - __x + _Tp(1.5L))));
const _Tp __mu = __nu - __nl;
const _Tp __mu2 = __mu * __mu;
const _Tp __xi = _Tp(1) / __x;
const _Tp __xi2 = _Tp(2) * __xi;
_Tp __w = __xi2 / __numeric_constants<_Tp>::__pi();
int __isign = 1;
_Tp __h = __nu * __xi;
if (__h < __fp_min)
__h = __fp_min;
_Tp __b = __xi2 * __nu;
_Tp __d = _Tp(0);
_Tp __c = __h;
int __i;
for (__i = 1; __i <= __max_iter; ++__i)
{
__b += __xi2;
__d = __b - __d;
if (std::abs(__d) < __fp_min)
__d = __fp_min;
__c = __b - _Tp(1) / __c;
if (std::abs(__c) < __fp_min)
__c = __fp_min;
__d = _Tp(1) / __d;
const _Tp __del = __c * __d;
__h *= __del;
if (__d < _Tp(0))
__isign = -__isign;
if (std::abs(__del - _Tp(1)) < __eps)
break;
}
if (__i > __max_iter)
std::__throw_runtime_error(("Argument x too large in __bessel_jn; " "try asymptotic expansion.")
);
_Tp __Jnul = __isign * __fp_min;
_Tp __Jpnul = __h * __Jnul;
_Tp __Jnul1 = __Jnul;
_Tp __Jpnu1 = __Jpnul;
_Tp __fact = __nu * __xi;
for ( int __l = __nl; __l >= 1; --__l )
{
const _Tp __Jnutemp = __fact * __Jnul + __Jpnul;
__fact -= __xi;
__Jpnul = __fact * __Jnutemp - __Jnul;
__Jnul = __Jnutemp;
}
if (__Jnul == _Tp(0))
__Jnul = __eps;
_Tp __f= __Jpnul / __Jnul;
_Tp __Nmu, __Nnu1, __Npmu, __Jmu;
if (__x < __x_min)
{
const _Tp __x2 = __x / _Tp(2);
const _Tp __pimu = __numeric_constants<_Tp>::__pi() * __mu;
_Tp __fact = (std::abs(__pimu) < __eps
? _Tp(1) : __pimu / std::sin(__pimu));
_Tp __d = -std::log(__x2);
_Tp __e = __mu * __d;
_Tp __fact2 = (std::abs(__e) < __eps
? _Tp(1) : std::sinh(__e) / __e);
_Tp __gam1, __gam2, __gampl, __gammi;
__gamma_temme(__mu, __gam1, __gam2, __gampl, __gammi);
_Tp __ff = (_Tp(2) / __numeric_constants<_Tp>::__pi())
* __fact * (__gam1 * std::cosh(__e) + __gam2 * __fact2 * __d);
__e = std::exp(__e);
_Tp __p = __e / (__numeric_constants<_Tp>::__pi() * __gampl);
_Tp __q = _Tp(1) / (__e * __numeric_constants<_Tp>::__pi() * __gammi);
const _Tp __pimu2 = __pimu / _Tp(2);
_Tp __fact3 = (std::abs(__pimu2) < __eps
? _Tp(1) : std::sin(__pimu2) / __pimu2 );
_Tp __r = __numeric_constants<_Tp>::__pi() * __pimu2 * __fact3 * __fact3;
_Tp __c = _Tp(1);
__d = -__x2 * __x2;
_Tp __sum = __ff + __r * __q;
_Tp __sum1 = __p;
for (__i = 1; __i <= __max_iter; ++__i)
{
__ff = (__i * __ff + __p + __q) / (__i * __i - __mu2);
__c *= __d / _Tp(__i);
__p /= _Tp(__i) - __mu;
__q /= _Tp(__i) + __mu;
const _Tp __del = __c * (__ff + __r * __q);
__sum += __del;
const _Tp __del1 = __c * __p - __i * __del;
__sum1 += __del1;
if ( std::abs(__del) < __eps * (_Tp(1) + std::abs(__sum)) )
break;
}
if ( __i > __max_iter )
std::__throw_runtime_error(("Bessel y series failed to converge " "in __bessel_jn.")
);
__Nmu = -__sum;
__Nnu1 = -__sum1 * __xi2;
__Npmu = __mu * __xi * __Nmu - __Nnu1;
__Jmu = __w / (__Npmu - __f * __Nmu);
}
else
{
_Tp __a = _Tp(0.25L) - __mu2;
_Tp __q = _Tp(1);
_Tp __p = -__xi / _Tp(2);
_Tp __br = _Tp(2) * __x;
_Tp __bi = _Tp(2);
_Tp __fact = __a * __xi / (__p * __p + __q * __q);
_Tp __cr = __br + __q * __fact;
_Tp __ci = __bi + __p * __fact;
_Tp __den = __br * __br + __bi * __bi;
_Tp __dr = __br / __den;
_Tp __di = -__bi / __den;
_Tp __dlr = __cr * __dr - __ci * __di;
_Tp __dli = __cr * __di + __ci * __dr;
_Tp __temp = __p * __dlr - __q * __dli;
__q = __p * __dli + __q * __dlr;
__p = __temp;
int __i;
for (__i = 2; __i <= __max_iter; ++__i)
{
__a += _Tp(2 * (__i - 1));
__bi += _Tp(2);
__dr = __a * __dr + __br;
__di = __a * __di + __bi;
if (std::abs(__dr) + std::abs(__di) < __fp_min)
__dr = __fp_min;
__fact = __a / (__cr * __cr + __ci * __ci);
__cr = __br + __cr * __fact;
__ci = __bi - __ci * __fact;
if (std::abs(__cr) + std::abs(__ci) < __fp_min)
__cr = __fp_min;
__den = __dr * __dr + __di * __di;
__dr /= __den;
__di /= -__den;
__dlr = __cr * __dr - __ci * __di;
__dli = __cr * __di + __ci * __dr;
__temp = __p * __dlr - __q * __dli;
__q = __p * __dli + __q * __dlr;
__p = __temp;
if (std::abs(__dlr - _Tp(1)) + std::abs(__dli) < __eps)
break;
}
if (__i > __max_iter)
std::__throw_runtime_error(("Lentz's method failed " "in __bessel_jn.")
);
const _Tp __gam = (__p - __f) / __q;
__Jmu = std::sqrt(__w / ((__p - __f) * __gam + __q));
__Jmu = ::std::copysign(__Jmu, __Jnul);
__Nmu = __gam * __Jmu;
__Npmu = (__p + __q / __gam) * __Nmu;
__Nnu1 = __mu * __xi * __Nmu - __Npmu;
}
__fact = __Jmu / __Jnul;
__Jnu = __fact * __Jnul1;
__Jpnu = __fact * __Jpnu1;
for (__i = 1; __i <= __nl; ++__i)
{
const _Tp __Nnutemp = (__mu + __i) * __xi2 * __Nnu1 - __Nmu;
__Nmu = __Nnu1;
__Nnu1 = __Nnutemp;
}
__Nnu = __Nmu;
__Npnu = __nu * __xi * __Nmu - __Nnu1;
return;
}
template <typename _Tp>
void
__cyl_bessel_jn_asymp(_Tp __nu, _Tp __x, _Tp & __Jnu, _Tp & __Nnu)
{
const _Tp __mu = _Tp(4) * __nu * __nu;
const _Tp __8x = _Tp(8) * __x;
_Tp __P = _Tp(0);
_Tp __Q = _Tp(0);
_Tp __k = _Tp(0);
_Tp __term = _Tp(1);
int __epsP = 0;
int __epsQ = 0;
_Tp __eps = std::numeric_limits<_Tp>::epsilon();
do
{
__term *= (__k == 0
? _Tp(1)
: -(__mu - (2 * __k - 1) * (2 * __k - 1)) / (__k * __8x));
__epsP = std::abs(__term) < __eps * std::abs(__P);
__P += __term;
__k++;
__term *= (__mu - (2 * __k - 1) * (2 * __k - 1)) / (__k * __8x);
__epsQ = std::abs(__term) < __eps * std::abs(__Q);
__Q += __term;
if (__epsP && __epsQ && __k > (__nu / 2.))
break;
__k++;
}
while (__k < 1000);
const _Tp __chi = __x - (__nu + _Tp(0.5L))
* __numeric_constants<_Tp>::__pi_2();
const _Tp __c = std::cos(__chi);
const _Tp __s = std::sin(__chi);
const _Tp __coef = std::sqrt(_Tp(2)
/ (__numeric_constants<_Tp>::__pi() * __x));
__Jnu = __coef * (__c * __P - __s * __Q);
__Nnu = __coef * (__s * __P + __c * __Q);
return;
}
template <typename _Tp>
_Tp
__cyl_bessel_ij_series(_Tp __nu, _Tp __x, _Tp __sgn,
unsigned int __max_iter)
{
if (__x == _Tp(0))
return __nu == _Tp(0) ? _Tp(1) : _Tp(0);
const _Tp __x2 = __x / _Tp(2);
_Tp __fact = __nu * std::log(__x2);
__fact -= ::std::lgamma(__nu + _Tp(1));
__fact = std::exp(__fact);
const _Tp __xx4 = __sgn * __x2 * __x2;
_Tp __Jn = _Tp(1);
_Tp __term = _Tp(1);
for (unsigned int __i = 1; __i < __max_iter; ++__i)
{
__term *= __xx4 / (_Tp(__i) * (__nu + _Tp(__i)));
__Jn += __term;
if (std::abs(__term / __Jn) < std::numeric_limits<_Tp>::epsilon())
break;
}
return __fact * __Jn;
}
template<typename _Tp>
_Tp
__cyl_bessel_j(_Tp __nu, _Tp __x)
{
if (__nu < _Tp(0) || __x < _Tp(0))
std::__throw_domain_error(("Bad argument " "in __cyl_bessel_j.")
);
else if (__isnan(__nu) || __isnan(__x))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (__x * __x < _Tp(10) * (__nu + _Tp(1)))
return __cyl_bessel_ij_series(__nu, __x, -_Tp(1), 200);
else if (__x > _Tp(1000))
{
_Tp __J_nu, __N_nu;
__cyl_bessel_jn_asymp(__nu, __x, __J_nu, __N_nu);
return __J_nu;
}
else
{
_Tp __J_nu, __N_nu, __Jp_nu, __Np_nu;
__bessel_jn(__nu, __x, __J_nu, __N_nu, __Jp_nu, __Np_nu);
return __J_nu;
}
}
template<typename _Tp>
_Tp
__cyl_neumann_n(_Tp __nu, _Tp __x)
{
if (__nu < _Tp(0) || __x < _Tp(0))
std::__throw_domain_error(("Bad argument " "in __cyl_neumann_n.")
);
else if (__isnan(__nu) || __isnan(__x))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (__x > _Tp(1000))
{
_Tp __J_nu, __N_nu;
__cyl_bessel_jn_asymp(__nu, __x, __J_nu, __N_nu);
return __N_nu;
}
else
{
_Tp __J_nu, __N_nu, __Jp_nu, __Np_nu;
__bessel_jn(__nu, __x, __J_nu, __N_nu, __Jp_nu, __Np_nu);
return __N_nu;
}
}
template <typename _Tp>
void
__sph_bessel_jn(unsigned int __n, _Tp __x,
_Tp & __j_n, _Tp & __n_n, _Tp & __jp_n, _Tp & __np_n)
{
const _Tp __nu = _Tp(__n) + _Tp(0.5L);
_Tp __J_nu, __N_nu, __Jp_nu, __Np_nu;
__bessel_jn(__nu, __x, __J_nu, __N_nu, __Jp_nu, __Np_nu);
const _Tp __factor = __numeric_constants<_Tp>::__sqrtpio2()
/ std::sqrt(__x);
__j_n = __factor * __J_nu;
__n_n = __factor * __N_nu;
__jp_n = __factor * __Jp_nu - __j_n / (_Tp(2) * __x);
__np_n = __factor * __Np_nu - __n_n / (_Tp(2) * __x);
return;
}
template <typename _Tp>
_Tp
__sph_bessel(unsigned int __n, _Tp __x)
{
if (__x < _Tp(0))
std::__throw_domain_error(("Bad argument " "in __sph_bessel.")
);
else if (__isnan(__x))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (__x == _Tp(0))
{
if (__n == 0)
return _Tp(1);
else
return _Tp(0);
}
else
{
_Tp __j_n, __n_n, __jp_n, __np_n;
__sph_bessel_jn(__n, __x, __j_n, __n_n, __jp_n, __np_n);
return __j_n;
}
}
template <typename _Tp>
_Tp
__sph_neumann(unsigned int __n, _Tp __x)
{
if (__x < _Tp(0))
std::__throw_domain_error(("Bad argument " "in __sph_neumann.")
);
else if (__isnan(__x))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (__x == _Tp(0))
return -std::numeric_limits<_Tp>::infinity();
else
{
_Tp __j_n, __n_n, __jp_n, __np_n;
__sph_bessel_jn(__n, __x, __j_n, __n_n, __jp_n, __np_n);
return __n_n;
}
}
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __detail
{
template<typename _Tp>
_Tp
__beta_gamma(_Tp __x, _Tp __y)
{
_Tp __bet;
if (__x > __y)
{
__bet = ::std::tgamma(__x)
/ ::std::tgamma(__x + __y);
__bet *= ::std::tgamma(__y);
}
else
{
__bet = ::std::tgamma(__y)
/ ::std::tgamma(__x + __y);
__bet *= ::std::tgamma(__x);
}
return __bet;
}
template<typename _Tp>
_Tp
__beta_lgamma(_Tp __x, _Tp __y)
{
_Tp __bet = ::std::lgamma(__x)
+ ::std::lgamma(__y)
- ::std::lgamma(__x + __y);
__bet = std::exp(__bet);
return __bet;
}
template<typename _Tp>
_Tp
__beta_product(_Tp __x, _Tp __y)
{
_Tp __bet = (__x + __y) / (__x * __y);
unsigned int __max_iter = 1000000;
for (unsigned int __k = 1; __k < __max_iter; ++__k)
{
_Tp __term = (_Tp(1) + (__x + __y) / __k)
/ ((_Tp(1) + __x / __k) * (_Tp(1) + __y / __k));
__bet *= __term;
}
return __bet;
}
template<typename _Tp>
inline _Tp
__beta(_Tp __x, _Tp __y)
{
if (__isnan(__x) || __isnan(__y))
return std::numeric_limits<_Tp>::quiet_NaN();
else
return __beta_lgamma(__x, __y);
}
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __detail
{
template<typename _Tp>
_Tp
__ellint_rf(_Tp __x, _Tp __y, _Tp __z)
{
const _Tp __min = std::numeric_limits<_Tp>::min();
const _Tp __lolim = _Tp(5) * __min;
if (__x < _Tp(0) || __y < _Tp(0) || __z < _Tp(0))
std::__throw_domain_error(("Argument less than zero " "in __ellint_rf.")
);
else if (__x + __y < __lolim || __x + __z < __lolim
|| __y + __z < __lolim)
std::__throw_domain_error(("Argument too small in __ellint_rf"));
else
{
const _Tp __c0 = _Tp(1) / _Tp(4);
const _Tp __c1 = _Tp(1) / _Tp(24);
const _Tp __c2 = _Tp(1) / _Tp(10);
const _Tp __c3 = _Tp(3) / _Tp(44);
const _Tp __c4 = _Tp(1) / _Tp(14);
_Tp __xn = __x;
_Tp __yn = __y;
_Tp __zn = __z;
const _Tp __eps = std::numeric_limits<_Tp>::epsilon();
const _Tp __errtol = std::pow(__eps, _Tp(1) / _Tp(6));
_Tp __mu;
_Tp __xndev, __yndev, __zndev;
const unsigned int __max_iter = 100;
for (unsigned int __iter = 0; __iter < __max_iter; ++__iter)
{
__mu = (__xn + __yn + __zn) / _Tp(3);
__xndev = 2 - (__mu + __xn) / __mu;
__yndev = 2 - (__mu + __yn) / __mu;
__zndev = 2 - (__mu + __zn) / __mu;
_Tp __epsilon = std::max(std::abs(__xndev), std::abs(__yndev));
__epsilon = std::max(__epsilon, std::abs(__zndev));
if (__epsilon < __errtol)
break;
const _Tp __xnroot = std::sqrt(__xn);
const _Tp __ynroot = std::sqrt(__yn);
const _Tp __znroot = std::sqrt(__zn);
const _Tp __lambda = __xnroot * (__ynroot + __znroot)
+ __ynroot * __znroot;
__xn = __c0 * (__xn + __lambda);
__yn = __c0 * (__yn + __lambda);
__zn = __c0 * (__zn + __lambda);
}
const _Tp __e2 = __xndev * __yndev - __zndev * __zndev;
const _Tp __e3 = __xndev * __yndev * __zndev;
const _Tp __s = _Tp(1) + (__c1 * __e2 - __c2 - __c3 * __e3) * __e2
+ __c4 * __e3;
return __s / std::sqrt(__mu);
}
}
template<typename _Tp>
_Tp
__comp_ellint_1_series(_Tp __k)
{
const _Tp __kk = __k * __k;
_Tp __term = __kk / _Tp(4);
_Tp __sum = _Tp(1) + __term;
const unsigned int __max_iter = 1000;
for (unsigned int __i = 2; __i < __max_iter; ++__i)
{
__term *= (2 * __i - 1) * __kk / (2 * __i);
if (__term < std::numeric_limits<_Tp>::epsilon())
break;
__sum += __term;
}
return __numeric_constants<_Tp>::__pi_2() * __sum;
}
template<typename _Tp>
_Tp
__comp_ellint_1(_Tp __k)
{
if (__isnan(__k))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (std::abs(__k) >= _Tp(1))
return std::numeric_limits<_Tp>::quiet_NaN();
else
return __ellint_rf(_Tp(0), _Tp(1) - __k * __k, _Tp(1));
}
template<typename _Tp>
_Tp
__ellint_1(_Tp __k, _Tp __phi)
{
if (__isnan(__k) || __isnan(__phi))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (std::abs(__k) > _Tp(1))
std::__throw_domain_error(("Bad argument in __ellint_1."));
else
{
const int __n = std::floor(__phi / __numeric_constants<_Tp>::__pi()
+ _Tp(0.5L));
const _Tp __phi_red = __phi
- __n * __numeric_constants<_Tp>::__pi();
const _Tp __s = std::sin(__phi_red);
const _Tp __c = std::cos(__phi_red);
const _Tp __F = __s
* __ellint_rf(__c * __c,
_Tp(1) - __k * __k * __s * __s, _Tp(1));
if (__n == 0)
return __F;
else
return __F + _Tp(2) * __n * __comp_ellint_1(__k);
}
}
template<typename _Tp>
_Tp
__comp_ellint_2_series(_Tp __k)
{
const _Tp __kk = __k * __k;
_Tp __term = __kk;
_Tp __sum = __term;
const unsigned int __max_iter = 1000;
for (unsigned int __i = 2; __i < __max_iter; ++__i)
{
const _Tp __i2m = 2 * __i - 1;
const _Tp __i2 = 2 * __i;
__term *= __i2m * __i2m * __kk / (__i2 * __i2);
if (__term < std::numeric_limits<_Tp>::epsilon())
break;
__sum += __term / __i2m;
}
return __numeric_constants<_Tp>::__pi_2() * (_Tp(1) - __sum);
}
template<typename _Tp>
_Tp
__ellint_rd(_Tp __x, _Tp __y, _Tp __z)
{
const _Tp __eps = std::numeric_limits<_Tp>::epsilon();
const _Tp __errtol = std::pow(__eps / _Tp(8), _Tp(1) / _Tp(6));
const _Tp __max = std::numeric_limits<_Tp>::max();
const _Tp __lolim = _Tp(2) / std::pow(__max, _Tp(2) / _Tp(3));
if (__x < _Tp(0) || __y < _Tp(0))
std::__throw_domain_error(("Argument less than zero " "in __ellint_rd.")
);
else if (__x + __y < __lolim || __z < __lolim)
std::__throw_domain_error(("Argument too small " "in __ellint_rd.")
);
else
{
const _Tp __c0 = _Tp(1) / _Tp(4);
const _Tp __c1 = _Tp(3) / _Tp(14);
const _Tp __c2 = _Tp(1) / _Tp(6);
const _Tp __c3 = _Tp(9) / _Tp(22);
const _Tp __c4 = _Tp(3) / _Tp(26);
_Tp __xn = __x;
_Tp __yn = __y;
_Tp __zn = __z;
_Tp __sigma = _Tp(0);
_Tp __power4 = _Tp(1);
_Tp __mu;
_Tp __xndev, __yndev, __zndev;
const unsigned int __max_iter = 100;
for (unsigned int __iter = 0; __iter < __max_iter; ++__iter)
{
__mu = (__xn + __yn + _Tp(3) * __zn) / _Tp(5);
__xndev = (__mu - __xn) / __mu;
__yndev = (__mu - __yn) / __mu;
__zndev = (__mu - __zn) / __mu;
_Tp __epsilon = std::max(std::abs(__xndev), std::abs(__yndev));
__epsilon = std::max(__epsilon, std::abs(__zndev));
if (__epsilon < __errtol)
break;
_Tp __xnroot = std::sqrt(__xn);
_Tp __ynroot = std::sqrt(__yn);
_Tp __znroot = std::sqrt(__zn);
_Tp __lambda = __xnroot * (__ynroot + __znroot)
+ __ynroot * __znroot;
__sigma += __power4 / (__znroot * (__zn + __lambda));
__power4 *= __c0;
__xn = __c0 * (__xn + __lambda);
__yn = __c0 * (__yn + __lambda);
__zn = __c0 * (__zn + __lambda);
}
_Tp __ea = __xndev * __yndev;
_Tp __eb = __zndev * __zndev;
_Tp __ec = __ea - __eb;
_Tp __ed = __ea - _Tp(6) * __eb;
_Tp __ef = __ed + __ec + __ec;
_Tp __s1 = __ed * (-__c1 + __c3 * __ed
/ _Tp(3) - _Tp(3) * __c4 * __zndev * __ef
/ _Tp(2));
_Tp __s2 = __zndev
* (__c2 * __ef
+ __zndev * (-__c3 * __ec - __zndev * __c4 - __ea));
return _Tp(3) * __sigma + __power4 * (_Tp(1) + __s1 + __s2)
/ (__mu * std::sqrt(__mu));
}
}
template<typename _Tp>
_Tp
__comp_ellint_2(_Tp __k)
{
if (__isnan(__k))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (std::abs(__k) == 1)
return _Tp(1);
else if (std::abs(__k) > _Tp(1))
std::__throw_domain_error(("Bad argument in __comp_ellint_2."));
else
{
const _Tp __kk = __k * __k;
return __ellint_rf(_Tp(0), _Tp(1) - __kk, _Tp(1))
- __kk * __ellint_rd(_Tp(0), _Tp(1) - __kk, _Tp(1)) / _Tp(3);
}
}
template<typename _Tp>
_Tp
__ellint_2(_Tp __k, _Tp __phi)
{
if (__isnan(__k) || __isnan(__phi))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (std::abs(__k) > _Tp(1))
std::__throw_domain_error(("Bad argument in __ellint_2."));
else
{
const int __n = std::floor(__phi / __numeric_constants<_Tp>::__pi()
+ _Tp(0.5L));
const _Tp __phi_red = __phi
- __n * __numeric_constants<_Tp>::__pi();
const _Tp __kk = __k * __k;
const _Tp __s = std::sin(__phi_red);
const _Tp __ss = __s * __s;
const _Tp __sss = __ss * __s;
const _Tp __c = std::cos(__phi_red);
const _Tp __cc = __c * __c;
const _Tp __E = __s
* __ellint_rf(__cc, _Tp(1) - __kk * __ss, _Tp(1))
- __kk * __sss
* __ellint_rd(__cc, _Tp(1) - __kk * __ss, _Tp(1))
/ _Tp(3);
if (__n == 0)
return __E;
else
return __E + _Tp(2) * __n * __comp_ellint_2(__k);
}
}
template<typename _Tp>
_Tp
__ellint_rc(_Tp __x, _Tp __y)
{
const _Tp __min = std::numeric_limits<_Tp>::min();
const _Tp __lolim = _Tp(5) * __min;
if (__x < _Tp(0) || __y < _Tp(0) || __x + __y < __lolim)
std::__throw_domain_error(("Argument less than zero " "in __ellint_rc.")
);
else
{
const _Tp __c0 = _Tp(1) / _Tp(4);
const _Tp __c1 = _Tp(1) / _Tp(7);
const _Tp __c2 = _Tp(9) / _Tp(22);
const _Tp __c3 = _Tp(3) / _Tp(10);
const _Tp __c4 = _Tp(3) / _Tp(8);
_Tp __xn = __x;
_Tp __yn = __y;
const _Tp __eps = std::numeric_limits<_Tp>::epsilon();
const _Tp __errtol = std::pow(__eps / _Tp(30), _Tp(1) / _Tp(6));
_Tp __mu;
_Tp __sn;
const unsigned int __max_iter = 100;
for (unsigned int __iter = 0; __iter < __max_iter; ++__iter)
{
__mu = (__xn + _Tp(2) * __yn) / _Tp(3);
__sn = (__yn + __mu) / __mu - _Tp(2);
if (std::abs(__sn) < __errtol)
break;
const _Tp __lambda = _Tp(2) * std::sqrt(__xn) * std::sqrt(__yn)
+ __yn;
__xn = __c0 * (__xn + __lambda);
__yn = __c0 * (__yn + __lambda);
}
_Tp __s = __sn * __sn
* (__c3 + __sn*(__c1 + __sn * (__c4 + __sn * __c2)));
return (_Tp(1) + __s) / std::sqrt(__mu);
}
}
template<typename _Tp>
_Tp
__ellint_rj(_Tp __x, _Tp __y, _Tp __z, _Tp __p)
{
const _Tp __min = std::numeric_limits<_Tp>::min();
const _Tp __lolim = std::pow(_Tp(5) * __min, _Tp(1)/_Tp(3));
if (__x < _Tp(0) || __y < _Tp(0) || __z < _Tp(0))
std::__throw_domain_error(("Argument less than zero " "in __ellint_rj.")
);
else if (__x + __y < __lolim || __x + __z < __lolim
|| __y + __z < __lolim || __p < __lolim)
std::__throw_domain_error(("Argument too small " "in __ellint_rj")
);
else
{
const _Tp __c0 = _Tp(1) / _Tp(4);
const _Tp __c1 = _Tp(3) / _Tp(14);
const _Tp __c2 = _Tp(1) / _Tp(3);
const _Tp __c3 = _Tp(3) / _Tp(22);
const _Tp __c4 = _Tp(3) / _Tp(26);
_Tp __xn = __x;
_Tp __yn = __y;
_Tp __zn = __z;
_Tp __pn = __p;
_Tp __sigma = _Tp(0);
_Tp __power4 = _Tp(1);
const _Tp __eps = std::numeric_limits<_Tp>::epsilon();
const _Tp __errtol = std::pow(__eps / _Tp(8), _Tp(1) / _Tp(6));
_Tp __mu;
_Tp __xndev, __yndev, __zndev, __pndev;
const unsigned int __max_iter = 100;
for (unsigned int __iter = 0; __iter < __max_iter; ++__iter)
{
__mu = (__xn + __yn + __zn + _Tp(2) * __pn) / _Tp(5);
__xndev = (__mu - __xn) / __mu;
__yndev = (__mu - __yn) / __mu;
__zndev = (__mu - __zn) / __mu;
__pndev = (__mu - __pn) / __mu;
_Tp __epsilon = std::max(std::abs(__xndev), std::abs(__yndev));
__epsilon = std::max(__epsilon, std::abs(__zndev));
__epsilon = std::max(__epsilon, std::abs(__pndev));
if (__epsilon < __errtol)
break;
const _Tp __xnroot = std::sqrt(__xn);
const _Tp __ynroot = std::sqrt(__yn);
const _Tp __znroot = std::sqrt(__zn);
const _Tp __lambda = __xnroot * (__ynroot + __znroot)
+ __ynroot * __znroot;
const _Tp __alpha1 = __pn * (__xnroot + __ynroot + __znroot)
+ __xnroot * __ynroot * __znroot;
const _Tp __alpha2 = __alpha1 * __alpha1;
const _Tp __beta = __pn * (__pn + __lambda)
* (__pn + __lambda);
__sigma += __power4 * __ellint_rc(__alpha2, __beta);
__power4 *= __c0;
__xn = __c0 * (__xn + __lambda);
__yn = __c0 * (__yn + __lambda);
__zn = __c0 * (__zn + __lambda);
__pn = __c0 * (__pn + __lambda);
}
_Tp __ea = __xndev * (__yndev + __zndev) + __yndev * __zndev;
_Tp __eb = __xndev * __yndev * __zndev;
_Tp __ec = __pndev * __pndev;
_Tp __e2 = __ea - _Tp(3) * __ec;
_Tp __e3 = __eb + _Tp(2) * __pndev * (__ea - __ec);
_Tp __s1 = _Tp(1) + __e2 * (-__c1 + _Tp(3) * __c3 * __e2 / _Tp(4)
- _Tp(3) * __c4 * __e3 / _Tp(2));
_Tp __s2 = __eb * (__c2 / _Tp(2)
+ __pndev * (-__c3 - __c3 + __pndev * __c4));
_Tp __s3 = __pndev * __ea * (__c2 - __pndev * __c3)
- __c2 * __pndev * __ec;
return _Tp(3) * __sigma + __power4 * (__s1 + __s2 + __s3)
/ (__mu * std::sqrt(__mu));
}
}
template<typename _Tp>
_Tp
__comp_ellint_3(_Tp __k, _Tp __nu)
{
if (__isnan(__k) || __isnan(__nu))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (__nu == _Tp(1))
return std::numeric_limits<_Tp>::infinity();
else if (std::abs(__k) > _Tp(1))
std::__throw_domain_error(("Bad argument in __comp_ellint_3."));
else
{
const _Tp __kk = __k * __k;
return __ellint_rf(_Tp(0), _Tp(1) - __kk, _Tp(1))
+ __nu
* __ellint_rj(_Tp(0), _Tp(1) - __kk, _Tp(1), _Tp(1) - __nu)
/ _Tp(3);
}
}
template<typename _Tp>
_Tp
__ellint_3(_Tp __k, _Tp __nu, _Tp __phi)
{
if (__isnan(__k) || __isnan(__nu) || __isnan(__phi))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (std::abs(__k) > _Tp(1))
std::__throw_domain_error(("Bad argument in __ellint_3."));
else
{
const int __n = std::floor(__phi / __numeric_constants<_Tp>::__pi()
+ _Tp(0.5L));
const _Tp __phi_red = __phi
- __n * __numeric_constants<_Tp>::__pi();
const _Tp __kk = __k * __k;
const _Tp __s = std::sin(__phi_red);
const _Tp __ss = __s * __s;
const _Tp __sss = __ss * __s;
const _Tp __c = std::cos(__phi_red);
const _Tp __cc = __c * __c;
const _Tp __Pi = __s
* __ellint_rf(__cc, _Tp(1) - __kk * __ss, _Tp(1))
+ __nu * __sss
* __ellint_rj(__cc, _Tp(1) - __kk * __ss, _Tp(1),
_Tp(1) - __nu * __ss) / _Tp(3);
if (__n == 0)
return __Pi;
else
return __Pi + _Tp(2) * __n * __comp_ellint_3(__k, __nu);
}
}
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __detail
{
template<typename _Tp> _Tp __expint_E1(_Tp);
template<typename _Tp>
_Tp
__expint_E1_series(_Tp __x)
{
const _Tp __eps = std::numeric_limits<_Tp>::epsilon();
_Tp __term = _Tp(1);
_Tp __esum = _Tp(0);
_Tp __osum = _Tp(0);
const unsigned int __max_iter = 1000;
for (unsigned int __i = 1; __i < __max_iter; ++__i)
{
__term *= - __x / __i;
if (std::abs(__term) < __eps)
break;
if (__term >= _Tp(0))
__esum += __term / __i;
else
__osum += __term / __i;
}
return - __esum - __osum
- __numeric_constants<_Tp>::__gamma_e() - std::log(__x);
}
template<typename _Tp>
_Tp
__expint_E1_asymp(_Tp __x)
{
_Tp __term = _Tp(1);
_Tp __esum = _Tp(1);
_Tp __osum = _Tp(0);
const unsigned int __max_iter = 1000;
for (unsigned int __i = 1; __i < __max_iter; ++__i)
{
_Tp __prev = __term;
__term *= - __i / __x;
if (std::abs(__term) > std::abs(__prev))
break;
if (__term >= _Tp(0))
__esum += __term;
else
__osum += __term;
}
return std::exp(- __x) * (__esum + __osum) / __x;
}
template<typename _Tp>
_Tp
__expint_En_series(unsigned int __n, _Tp __x)
{
const unsigned int __max_iter = 1000;
const _Tp __eps = std::numeric_limits<_Tp>::epsilon();
const int __nm1 = __n - 1;
_Tp __ans = (__nm1 != 0
? _Tp(1) / __nm1 : -std::log(__x)
- __numeric_constants<_Tp>::__gamma_e());
_Tp __fact = _Tp(1);
for (int __i = 1; __i <= __max_iter; ++__i)
{
__fact *= -__x / _Tp(__i);
_Tp __del;
if ( __i != __nm1 )
__del = -__fact / _Tp(__i - __nm1);
else
{
_Tp __psi = -__numeric_constants<_Tp>::gamma_e();
for (int __ii = 1; __ii <= __nm1; ++__ii)
__psi += _Tp(1) / _Tp(__ii);
__del = __fact * (__psi - std::log(__x));
}
__ans += __del;
if (std::abs(__del) < __eps * std::abs(__ans))
return __ans;
}
std::__throw_runtime_error(("Series summation failed " "in __expint_En_series.")
);
}
template<typename _Tp>
_Tp
__expint_En_cont_frac(unsigned int __n, _Tp __x)
{
const unsigned int __max_iter = 1000;
const _Tp __eps = std::numeric_limits<_Tp>::epsilon();
const _Tp __fp_min = std::numeric_limits<_Tp>::min();
const int __nm1 = __n - 1;
_Tp __b = __x + _Tp(__n);
_Tp __c = _Tp(1) / __fp_min;
_Tp __d = _Tp(1) / __b;
_Tp __h = __d;
for ( unsigned int __i = 1; __i <= __max_iter; ++__i )
{
_Tp __a = -_Tp(__i * (__nm1 + __i));
__b += _Tp(2);
__d = _Tp(1) / (__a * __d + __b);
__c = __b + __a / __c;
const _Tp __del = __c * __d;
__h *= __del;
if (std::abs(__del - _Tp(1)) < __eps)
{
const _Tp __ans = __h * std::exp(-__x);
return __ans;
}
}
std::__throw_runtime_error(("Continued fraction failed " "in __expint_En_cont_frac.")
);
}
template<typename _Tp>
_Tp
__expint_En_recursion(unsigned int __n, _Tp __x)
{
_Tp __En;
_Tp __E1 = __expint_E1(__x);
if (__x < _Tp(__n))
{
__En = __E1;
for (unsigned int __j = 2; __j < __n; ++__j)
__En = (std::exp(-__x) - __x * __En) / _Tp(__j - 1);
}
else
{
__En = _Tp(1);
const int __N = __n + 20;
_Tp __save = _Tp(0);
for (int __j = __N; __j > 0; --__j)
{
__En = (std::exp(-__x) - __j * __En) / __x;
if (__j == __n)
__save = __En;
}
_Tp __norm = __En / __E1;
__En /= __norm;
}
return __En;
}
template<typename _Tp>
_Tp
__expint_Ei_series(_Tp __x)
{
_Tp __term = _Tp(1);
_Tp __sum = _Tp(0);
const unsigned int __max_iter = 1000;
for (unsigned int __i = 1; __i < __max_iter; ++__i)
{
__term *= __x / __i;
__sum += __term / __i;
if (__term < std::numeric_limits<_Tp>::epsilon() * __sum)
break;
}
return __numeric_constants<_Tp>::__gamma_e() + __sum + std::log(__x);
}
template<typename _Tp>
_Tp
__expint_Ei_asymp(_Tp __x)
{
_Tp __term = _Tp(1);
_Tp __sum = _Tp(1);
const unsigned int __max_iter = 1000;
for (unsigned int __i = 1; __i < __max_iter; ++__i)
{
_Tp __prev = __term;
__term *= __i / __x;
if (__term < std::numeric_limits<_Tp>::epsilon())
break;
if (__term >= __prev)
break;
__sum += __term;
}
return std::exp(__x) * __sum / __x;
}
template<typename _Tp>
_Tp
__expint_Ei(_Tp __x)
{
if (__x < _Tp(0))
return -__expint_E1(-__x);
else if (__x < -std::log(std::numeric_limits<_Tp>::epsilon()))
return __expint_Ei_series(__x);
else
return __expint_Ei_asymp(__x);
}
template<typename _Tp>
_Tp
__expint_E1(_Tp __x)
{
if (__x < _Tp(0))
return -__expint_Ei(-__x);
else if (__x < _Tp(1))
return __expint_E1_series(__x);
else if (__x < _Tp(100))
return __expint_En_cont_frac(1, __x);
else
return __expint_E1_asymp(__x);
}
template<typename _Tp>
_Tp
__expint_asymp(unsigned int __n, _Tp __x)
{
_Tp __term = _Tp(1);
_Tp __sum = _Tp(1);
for (unsigned int __i = 1; __i <= __n; ++__i)
{
_Tp __prev = __term;
__term *= -(__n - __i + 1) / __x;
if (std::abs(__term) > std::abs(__prev))
break;
__sum += __term;
}
return std::exp(-__x) * __sum / __x;
}
template<typename _Tp>
_Tp
__expint_large_n(unsigned int __n, _Tp __x)
{
const _Tp __xpn = __x + __n;
const _Tp __xpn2 = __xpn * __xpn;
_Tp __term = _Tp(1);
_Tp __sum = _Tp(1);
for (unsigned int __i = 1; __i <= __n; ++__i)
{
_Tp __prev = __term;
__term *= (__n - 2 * (__i - 1) * __x) / __xpn2;
if (std::abs(__term) < std::numeric_limits<_Tp>::epsilon())
break;
__sum += __term;
}
return std::exp(-__x) * __sum / __xpn;
}
template<typename _Tp>
_Tp
__expint(unsigned int __n, _Tp __x)
{
if (__isnan(__x))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (__n <= 1 && __x == _Tp(0))
return std::numeric_limits<_Tp>::infinity();
else
{
_Tp __E0 = std::exp(__x) / __x;
if (__n == 0)
return __E0;
_Tp __E1 = __expint_E1(__x);
if (__n == 1)
return __E1;
if (__x == _Tp(0))
return _Tp(1) / static_cast<_Tp>(__n - 1);
_Tp __En = __expint_En_recursion(__n, __x);
return __En;
}
}
template<typename _Tp>
inline _Tp
__expint(_Tp __x)
{
if (__isnan(__x))
return std::numeric_limits<_Tp>::quiet_NaN();
else
return __expint_Ei(__x);
}
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __detail
{
template<typename _Tp>
_Tp
__conf_hyperg_series(_Tp __a, _Tp __c, _Tp __x)
{
const _Tp __eps = std::numeric_limits<_Tp>::epsilon();
_Tp __term = _Tp(1);
_Tp __Fac = _Tp(1);
const unsigned int __max_iter = 100000;
unsigned int __i;
for (__i = 0; __i < __max_iter; ++__i)
{
__term *= (__a + _Tp(__i)) * __x
/ ((__c + _Tp(__i)) * _Tp(1 + __i));
if (std::abs(__term) < __eps)
{
break;
}
__Fac += __term;
}
if (__i == __max_iter)
std::__throw_runtime_error(("Series failed to converge " "in __conf_hyperg_series.")
);
return __Fac;
}
template<typename _Tp>
_Tp
__conf_hyperg_luke(_Tp __a, _Tp __c, _Tp __xin)
{
const _Tp __big = std::pow(std::numeric_limits<_Tp>::max(), _Tp(0.16L));
const int __nmax = 20000;
const _Tp __eps = std::numeric_limits<_Tp>::epsilon();
const _Tp __x = -__xin;
const _Tp __x3 = __x * __x * __x;
const _Tp __t0 = __a / __c;
const _Tp __t1 = (__a + _Tp(1)) / (_Tp(2) * __c);
const _Tp __t2 = (__a + _Tp(2)) / (_Tp(2) * (__c + _Tp(1)));
_Tp __F = _Tp(1);
_Tp __prec;
_Tp __Bnm3 = _Tp(1);
_Tp __Bnm2 = _Tp(1) + __t1 * __x;
_Tp __Bnm1 = _Tp(1) + __t2 * __x * (_Tp(1) + __t1 / _Tp(3) * __x);
_Tp __Anm3 = _Tp(1);
_Tp __Anm2 = __Bnm2 - __t0 * __x;
_Tp __Anm1 = __Bnm1 - __t0 * (_Tp(1) + __t2 * __x) * __x
+ __t0 * __t1 * (__c / (__c + _Tp(1))) * __x * __x;
int __n = 3;
while(1)
{
_Tp __npam1 = _Tp(__n - 1) + __a;
_Tp __npcm1 = _Tp(__n - 1) + __c;
_Tp __npam2 = _Tp(__n - 2) + __a;
_Tp __npcm2 = _Tp(__n - 2) + __c;
_Tp __tnm1 = _Tp(2 * __n - 1);
_Tp __tnm3 = _Tp(2 * __n - 3);
_Tp __tnm5 = _Tp(2 * __n - 5);
_Tp __F1 = (_Tp(__n - 2) - __a) / (_Tp(2) * __tnm3 * __npcm1);
_Tp __F2 = (_Tp(__n) + __a) * __npam1
/ (_Tp(4) * __tnm1 * __tnm3 * __npcm2 * __npcm1);
_Tp __F3 = -__npam2 * __npam1 * (_Tp(__n - 2) - __a)
/ (_Tp(8) * __tnm3 * __tnm3 * __tnm5
* (_Tp(__n - 3) + __c) * __npcm2 * __npcm1);
_Tp __E = -__npam1 * (_Tp(__n - 1) - __c)
/ (_Tp(2) * __tnm3 * __npcm2 * __npcm1);
_Tp __An = (_Tp(1) + __F1 * __x) * __Anm1
+ (__E + __F2 * __x) * __x * __Anm2 + __F3 * __x3 * __Anm3;
_Tp __Bn = (_Tp(1) + __F1 * __x) * __Bnm1
+ (__E + __F2 * __x) * __x * __Bnm2 + __F3 * __x3 * __Bnm3;
_Tp __r = __An / __Bn;
__prec = std::abs((__F - __r) / __F);
__F = __r;
if (__prec < __eps || __n > __nmax)
break;
if (std::abs(__An) > __big || std::abs(__Bn) > __big)
{
__An /= __big;
__Bn /= __big;
__Anm1 /= __big;
__Bnm1 /= __big;
__Anm2 /= __big;
__Bnm2 /= __big;
__Anm3 /= __big;
__Bnm3 /= __big;
}
else if (std::abs(__An) < _Tp(1) / __big
|| std::abs(__Bn) < _Tp(1) / __big)
{
__An *= __big;
__Bn *= __big;
__Anm1 *= __big;
__Bnm1 *= __big;
__Anm2 *= __big;
__Bnm2 *= __big;
__Anm3 *= __big;
__Bnm3 *= __big;
}
++__n;
__Bnm3 = __Bnm2;
__Bnm2 = __Bnm1;
__Bnm1 = __Bn;
__Anm3 = __Anm2;
__Anm2 = __Anm1;
__Anm1 = __An;
}
if (__n >= __nmax)
std::__throw_runtime_error(("Iteration failed to converge " "in __conf_hyperg_luke.")
);
return __F;
}
template<typename _Tp>
_Tp
__conf_hyperg(_Tp __a, _Tp __c, _Tp __x)
{
const _Tp __c_nint = ::std::nearbyint(__c);
if (__isnan(__a) || __isnan(__c) || __isnan(__x))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (__c_nint == __c && __c_nint <= 0)
return std::numeric_limits<_Tp>::infinity();
else if (__a == _Tp(0))
return _Tp(1);
else if (__c == __a)
return std::exp(__x);
else if (__x < _Tp(0))
return __conf_hyperg_luke(__a, __c, __x);
else
return __conf_hyperg_series(__a, __c, __x);
}
template<typename _Tp>
_Tp
__hyperg_series(_Tp __a, _Tp __b, _Tp __c, _Tp __x)
{
const _Tp __eps = std::numeric_limits<_Tp>::epsilon();
_Tp __term = _Tp(1);
_Tp __Fabc = _Tp(1);
const unsigned int __max_iter = 100000;
unsigned int __i;
for (__i = 0; __i < __max_iter; ++__i)
{
__term *= (__a + _Tp(__i)) * (__b + _Tp(__i)) * __x
/ ((__c + _Tp(__i)) * _Tp(1 + __i));
if (std::abs(__term) < __eps)
{
break;
}
__Fabc += __term;
}
if (__i == __max_iter)
std::__throw_runtime_error(("Series failed to converge " "in __hyperg_series.")
);
return __Fabc;
}
template<typename _Tp>
_Tp
__hyperg_luke(_Tp __a, _Tp __b, _Tp __c, _Tp __xin)
{
const _Tp __big = std::pow(std::numeric_limits<_Tp>::max(), _Tp(0.16L));
const int __nmax = 20000;
const _Tp __eps = std::numeric_limits<_Tp>::epsilon();
const _Tp __x = -__xin;
const _Tp __x3 = __x * __x * __x;
const _Tp __t0 = __a * __b / __c;
const _Tp __t1 = (__a + _Tp(1)) * (__b + _Tp(1)) / (_Tp(2) * __c);
const _Tp __t2 = (__a + _Tp(2)) * (__b + _Tp(2))
/ (_Tp(2) * (__c + _Tp(1)));
_Tp __F = _Tp(1);
_Tp __Bnm3 = _Tp(1);
_Tp __Bnm2 = _Tp(1) + __t1 * __x;
_Tp __Bnm1 = _Tp(1) + __t2 * __x * (_Tp(1) + __t1 / _Tp(3) * __x);
_Tp __Anm3 = _Tp(1);
_Tp __Anm2 = __Bnm2 - __t0 * __x;
_Tp __Anm1 = __Bnm1 - __t0 * (_Tp(1) + __t2 * __x) * __x
+ __t0 * __t1 * (__c / (__c + _Tp(1))) * __x * __x;
int __n = 3;
while (1)
{
const _Tp __npam1 = _Tp(__n - 1) + __a;
const _Tp __npbm1 = _Tp(__n - 1) + __b;
const _Tp __npcm1 = _Tp(__n - 1) + __c;
const _Tp __npam2 = _Tp(__n - 2) + __a;
const _Tp __npbm2 = _Tp(__n - 2) + __b;
const _Tp __npcm2 = _Tp(__n - 2) + __c;
const _Tp __tnm1 = _Tp(2 * __n - 1);
const _Tp __tnm3 = _Tp(2 * __n - 3);
const _Tp __tnm5 = _Tp(2 * __n - 5);
const _Tp __n2 = __n * __n;
const _Tp __F1 = (_Tp(3) * __n2 + (__a + __b - _Tp(6)) * __n
+ _Tp(2) - __a * __b - _Tp(2) * (__a + __b))
/ (_Tp(2) * __tnm3 * __npcm1);
const _Tp __F2 = -(_Tp(3) * __n2 - (__a + __b + _Tp(6)) * __n
+ _Tp(2) - __a * __b) * __npam1 * __npbm1
/ (_Tp(4) * __tnm1 * __tnm3 * __npcm2 * __npcm1);
const _Tp __F3 = (__npam2 * __npam1 * __npbm2 * __npbm1
* (_Tp(__n - 2) - __a) * (_Tp(__n - 2) - __b))
/ (_Tp(8) * __tnm3 * __tnm3 * __tnm5
* (_Tp(__n - 3) + __c) * __npcm2 * __npcm1);
const _Tp __E = -__npam1 * __npbm1 * (_Tp(__n - 1) - __c)
/ (_Tp(2) * __tnm3 * __npcm2 * __npcm1);
_Tp __An = (_Tp(1) + __F1 * __x) * __Anm1
+ (__E + __F2 * __x) * __x * __Anm2 + __F3 * __x3 * __Anm3;
_Tp __Bn = (_Tp(1) + __F1 * __x) * __Bnm1
+ (__E + __F2 * __x) * __x * __Bnm2 + __F3 * __x3 * __Bnm3;
const _Tp __r = __An / __Bn;
const _Tp __prec = std::abs((__F - __r) / __F);
__F = __r;
if (__prec < __eps || __n > __nmax)
break;
if (std::abs(__An) > __big || std::abs(__Bn) > __big)
{
__An /= __big;
__Bn /= __big;
__Anm1 /= __big;
__Bnm1 /= __big;
__Anm2 /= __big;
__Bnm2 /= __big;
__Anm3 /= __big;
__Bnm3 /= __big;
}
else if (std::abs(__An) < _Tp(1) / __big
|| std::abs(__Bn) < _Tp(1) / __big)
{
__An *= __big;
__Bn *= __big;
__Anm1 *= __big;
__Bnm1 *= __big;
__Anm2 *= __big;
__Bnm2 *= __big;
__Anm3 *= __big;
__Bnm3 *= __big;
}
++__n;
__Bnm3 = __Bnm2;
__Bnm2 = __Bnm1;
__Bnm1 = __Bn;
__Anm3 = __Anm2;
__Anm2 = __Anm1;
__Anm1 = __An;
}
if (__n >= __nmax)
std::__throw_runtime_error(("Iteration failed to converge " "in __hyperg_luke.")
);
return __F;
}
template<typename _Tp>
_Tp
__hyperg_reflect(_Tp __a, _Tp __b, _Tp __c, _Tp __x)
{
const _Tp __d = __c - __a - __b;
const int __intd = std::floor(__d + _Tp(0.5L));
const _Tp __eps = std::numeric_limits<_Tp>::epsilon();
const _Tp __toler = _Tp(1000) * __eps;
const _Tp __log_max = std::log(std::numeric_limits<_Tp>::max());
const bool __d_integer = (std::abs(__d - __intd) < __toler);
if (__d_integer)
{
const _Tp __ln_omx = std::log(_Tp(1) - __x);
const _Tp __ad = std::abs(__d);
_Tp __F1, __F2;
_Tp __d1, __d2;
if (__d >= _Tp(0))
{
__d1 = __d;
__d2 = _Tp(0);
}
else
{
__d1 = _Tp(0);
__d2 = __d;
}
const _Tp __lng_c = __log_gamma(__c);
if (__ad < __eps)
{
__F1 = _Tp(0);
}
else
{
bool __ok_d1 = true;
_Tp __lng_ad, __lng_ad1, __lng_bd1;
try
{
__lng_ad = __log_gamma(__ad);
__lng_ad1 = __log_gamma(__a + __d1);
__lng_bd1 = __log_gamma(__b + __d1);
}
catch(...)
{
__ok_d1 = false;
}
if (__ok_d1)
{
_Tp __sum1 = _Tp(1);
_Tp __term = _Tp(1);
_Tp __ln_pre1 = __lng_ad + __lng_c + __d2 * __ln_omx
- __lng_ad1 - __lng_bd1;
for (int __i = 1; __i < __ad; ++__i)
{
const int __j = __i - 1;
__term *= (__a + __d2 + __j) * (__b + __d2 + __j)
/ (_Tp(1) + __d2 + __j) / __i * (_Tp(1) - __x);
__sum1 += __term;
}
if (__ln_pre1 > __log_max)
std::__throw_runtime_error(("Overflow of gamma functions" " in __hyperg_luke.")
);
else
__F1 = std::exp(__ln_pre1) * __sum1;
}
else
{
__F1 = _Tp(0);
}
}
bool __ok_d2 = true;
_Tp __lng_ad2, __lng_bd2;
try
{
__lng_ad2 = __log_gamma(__a + __d2);
__lng_bd2 = __log_gamma(__b + __d2);
}
catch(...)
{
__ok_d2 = false;
}
if (__ok_d2)
{
const int __maxiter = 2000;
const _Tp __psi_1 = -__numeric_constants<_Tp>::__gamma_e();
const _Tp __psi_1pd = __psi(_Tp(1) + __ad);
const _Tp __psi_apd1 = __psi(__a + __d1);
const _Tp __psi_bpd1 = __psi(__b + __d1);
_Tp __psi_term = __psi_1 + __psi_1pd - __psi_apd1
- __psi_bpd1 - __ln_omx;
_Tp __fact = _Tp(1);
_Tp __sum2 = __psi_term;
_Tp __ln_pre2 = __lng_c + __d1 * __ln_omx
- __lng_ad2 - __lng_bd2;
int __j;
for (__j = 1; __j < __maxiter; ++__j)
{
const _Tp __term1 = _Tp(1) / _Tp(__j)
+ _Tp(1) / (__ad + __j);
const _Tp __term2 = _Tp(1) / (__a + __d1 + _Tp(__j - 1))
+ _Tp(1) / (__b + __d1 + _Tp(__j - 1));
__psi_term += __term1 - __term2;
__fact *= (__a + __d1 + _Tp(__j - 1))
* (__b + __d1 + _Tp(__j - 1))
/ ((__ad + __j) * __j) * (_Tp(1) - __x);
const _Tp __delta = __fact * __psi_term;
__sum2 += __delta;
if (std::abs(__delta) < __eps * std::abs(__sum2))
break;
}
if (__j == __maxiter)
std::__throw_runtime_error(("Sum F2 failed to converge " "in __hyperg_reflect")
);
if (__sum2 == _Tp(0))
__F2 = _Tp(0);
else
__F2 = std::exp(__ln_pre2) * __sum2;
}
else
{
__F2 = _Tp(0);
}
const _Tp __sgn_2 = (__intd % 2 == 1 ? -_Tp(1) : _Tp(1));
const _Tp __F = __F1 + __sgn_2 * __F2;
return __F;
}
else
{
bool __ok1 = true;
_Tp __sgn_g1ca = _Tp(0), __ln_g1ca = _Tp(0);
_Tp __sgn_g1cb = _Tp(0), __ln_g1cb = _Tp(0);
try
{
__sgn_g1ca = __log_gamma_sign(__c - __a);
__ln_g1ca = __log_gamma(__c - __a);
__sgn_g1cb = __log_gamma_sign(__c - __b);
__ln_g1cb = __log_gamma(__c - __b);
}
catch(...)
{
__ok1 = false;
}
bool __ok2 = true;
_Tp __sgn_g2a = _Tp(0), __ln_g2a = _Tp(0);
_Tp __sgn_g2b = _Tp(0), __ln_g2b = _Tp(0);
try
{
__sgn_g2a = __log_gamma_sign(__a);
__ln_g2a = __log_gamma(__a);
__sgn_g2b = __log_gamma_sign(__b);
__ln_g2b = __log_gamma(__b);
}
catch(...)
{
__ok2 = false;
}
const _Tp __sgn_gc = __log_gamma_sign(__c);
const _Tp __ln_gc = __log_gamma(__c);
const _Tp __sgn_gd = __log_gamma_sign(__d);
const _Tp __ln_gd = __log_gamma(__d);
const _Tp __sgn_gmd = __log_gamma_sign(-__d);
const _Tp __ln_gmd = __log_gamma(-__d);
const _Tp __sgn1 = __sgn_gc * __sgn_gd * __sgn_g1ca * __sgn_g1cb;
const _Tp __sgn2 = __sgn_gc * __sgn_gmd * __sgn_g2a * __sgn_g2b;
_Tp __pre1, __pre2;
if (__ok1 && __ok2)
{
_Tp __ln_pre1 = __ln_gc + __ln_gd - __ln_g1ca - __ln_g1cb;
_Tp __ln_pre2 = __ln_gc + __ln_gmd - __ln_g2a - __ln_g2b
+ __d * std::log(_Tp(1) - __x);
if (__ln_pre1 < __log_max && __ln_pre2 < __log_max)
{
__pre1 = std::exp(__ln_pre1);
__pre2 = std::exp(__ln_pre2);
__pre1 *= __sgn1;
__pre2 *= __sgn2;
}
else
{
std::__throw_runtime_error(("Overflow of gamma functions " "in __hyperg_reflect")
);
}
}
else if (__ok1 && !__ok2)
{
_Tp __ln_pre1 = __ln_gc + __ln_gd - __ln_g1ca - __ln_g1cb;
if (__ln_pre1 < __log_max)
{
__pre1 = std::exp(__ln_pre1);
__pre1 *= __sgn1;
__pre2 = _Tp(0);
}
else
{
std::__throw_runtime_error(("Overflow of gamma functions " "in __hyperg_reflect")
);
}
}
else if (!__ok1 && __ok2)
{
_Tp __ln_pre2 = __ln_gc + __ln_gmd - __ln_g2a - __ln_g2b
+ __d * std::log(_Tp(1) - __x);
if (__ln_pre2 < __log_max)
{
__pre1 = _Tp(0);
__pre2 = std::exp(__ln_pre2);
__pre2 *= __sgn2;
}
else
{
std::__throw_runtime_error(("Overflow of gamma functions " "in __hyperg_reflect")
);
}
}
else
{
__pre1 = _Tp(0);
__pre2 = _Tp(0);
std::__throw_runtime_error(("Underflow of gamma functions " "in __hyperg_reflect")
);
}
const _Tp __F1 = __hyperg_series(__a, __b, _Tp(1) - __d,
_Tp(1) - __x);
const _Tp __F2 = __hyperg_series(__c - __a, __c - __b, _Tp(1) + __d,
_Tp(1) - __x);
const _Tp __F = __pre1 * __F1 + __pre2 * __F2;
return __F;
}
}
template<typename _Tp>
_Tp
__hyperg(_Tp __a, _Tp __b, _Tp __c, _Tp __x)
{
const _Tp __a_nint = ::std::nearbyint(__a);
const _Tp __b_nint = ::std::nearbyint(__b);
const _Tp __c_nint = ::std::nearbyint(__c);
const _Tp __toler = _Tp(1000) * std::numeric_limits<_Tp>::epsilon();
if (std::abs(__x) >= _Tp(1))
std::__throw_domain_error(("Argument outside unit circle " "in __hyperg.")
);
else if (__isnan(__a) || __isnan(__b)
|| __isnan(__c) || __isnan(__x))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (__c_nint == __c && __c_nint <= _Tp(0))
return std::numeric_limits<_Tp>::infinity();
else if (std::abs(__c - __b) < __toler || std::abs(__c - __a) < __toler)
return std::pow(_Tp(1) - __x, __c - __a - __b);
else if (__a >= _Tp(0) && __b >= _Tp(0) && __c >= _Tp(0)
&& __x >= _Tp(0) && __x < _Tp(0.995L))
return __hyperg_series(__a, __b, __c, __x);
else if (std::abs(__a) < _Tp(10) && std::abs(__b) < _Tp(10))
{
if (__a < _Tp(0) && std::abs(__a - __a_nint) < __toler)
return __hyperg_series(__a_nint, __b, __c, __x);
else if (__b < _Tp(0) && std::abs(__b - __b_nint) < __toler)
return __hyperg_series(__a, __b_nint, __c, __x);
else if (__x < -_Tp(0.25L))
return __hyperg_luke(__a, __b, __c, __x);
else if (__x < _Tp(0.5L))
return __hyperg_series(__a, __b, __c, __x);
else
if (std::abs(__c) > _Tp(10))
return __hyperg_series(__a, __b, __c, __x);
else
return __hyperg_reflect(__a, __b, __c, __x);
}
else
return __hyperg_luke(__a, __b, __c, __x);
}
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __detail
{
template<typename _Tp>
_Tp
__poly_legendre_p(unsigned int __l, _Tp __x)
{
if (__isnan(__x))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (__x == +_Tp(1))
return +_Tp(1);
else if (__x == -_Tp(1))
return (__l % 2 == 1 ? -_Tp(1) : +_Tp(1));
else
{
_Tp __p_lm2 = _Tp(1);
if (__l == 0)
return __p_lm2;
_Tp __p_lm1 = __x;
if (__l == 1)
return __p_lm1;
_Tp __p_l = 0;
for (unsigned int __ll = 2; __ll <= __l; ++__ll)
{
__p_l = _Tp(2) * __x * __p_lm1 - __p_lm2
- (__x * __p_lm1 - __p_lm2) / _Tp(__ll);
__p_lm2 = __p_lm1;
__p_lm1 = __p_l;
}
return __p_l;
}
}
template<typename _Tp>
_Tp
__assoc_legendre_p(unsigned int __l, unsigned int __m, _Tp __x,
_Tp __phase = _Tp(+1))
{
if (__m > __l)
return _Tp(0);
else if (__isnan(__x))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (__m == 0)
return __poly_legendre_p(__l, __x);
else
{
_Tp __p_mm = _Tp(1);
if (__m > 0)
{
_Tp __root = std::sqrt(_Tp(1) - __x) * std::sqrt(_Tp(1) + __x);
_Tp __fact = _Tp(1);
for (unsigned int __i = 1; __i <= __m; ++__i)
{
__p_mm *= __phase * __fact * __root;
__fact += _Tp(2);
}
}
if (__l == __m)
return __p_mm;
_Tp __p_mp1m = _Tp(2 * __m + 1) * __x * __p_mm;
if (__l == __m + 1)
return __p_mp1m;
_Tp __p_lm2m = __p_mm;
_Tp __P_lm1m = __p_mp1m;
_Tp __p_lm = _Tp(0);
for (unsigned int __j = __m + 2; __j <= __l; ++__j)
{
__p_lm = (_Tp(2 * __j - 1) * __x * __P_lm1m
- _Tp(__j + __m - 1) * __p_lm2m) / _Tp(__j - __m);
__p_lm2m = __P_lm1m;
__P_lm1m = __p_lm;
}
return __p_lm;
}
}
template <typename _Tp>
_Tp
__sph_legendre(unsigned int __l, unsigned int __m, _Tp __theta)
{
if (__isnan(__theta))
return std::numeric_limits<_Tp>::quiet_NaN();
const _Tp __x = std::cos(__theta);
if (__m > __l)
return _Tp(0);
else if (__m == 0)
{
_Tp __P = __poly_legendre_p(__l, __x);
_Tp __fact = std::sqrt(_Tp(2 * __l + 1)
/ (_Tp(4) * __numeric_constants<_Tp>::__pi()));
__P *= __fact;
return __P;
}
else if (__x == _Tp(1) || __x == -_Tp(1))
{
return _Tp(0);
}
else
{
const _Tp __sgn = ( __m % 2 == 1 ? -_Tp(1) : _Tp(1));
const _Tp __y_mp1m_factor = __x * std::sqrt(_Tp(2 * __m + 3));
const _Tp __lncirc = ::std::log1p(-__x * __x);
const _Tp __lnpoch = ::std::lgamma(_Tp(__m + _Tp(0.5L)))
- ::std::lgamma(_Tp(__m));
const _Tp __lnpre_val =
-_Tp(0.25L) * __numeric_constants<_Tp>::__lnpi()
+ _Tp(0.5L) * (__lnpoch + __m * __lncirc);
const _Tp __sr = std::sqrt((_Tp(2) + _Tp(1) / __m)
/ (_Tp(4) * __numeric_constants<_Tp>::__pi()));
_Tp __y_mm = __sgn * __sr * std::exp(__lnpre_val);
_Tp __y_mp1m = __y_mp1m_factor * __y_mm;
if (__l == __m)
return __y_mm;
else if (__l == __m + 1)
return __y_mp1m;
else
{
_Tp __y_lm = _Tp(0);
for (unsigned int __ll = __m + 2; __ll <= __l; ++__ll)
{
const _Tp __rat1 = _Tp(__ll - __m) / _Tp(__ll + __m);
const _Tp __rat2 = _Tp(__ll - __m - 1) / _Tp(__ll + __m - 1);
const _Tp __fact1 = std::sqrt(__rat1 * _Tp(2 * __ll + 1)
* _Tp(2 * __ll - 1));
const _Tp __fact2 = std::sqrt(__rat1 * __rat2 * _Tp(2 * __ll + 1)
/ _Tp(2 * __ll - 3));
__y_lm = (__x * __y_mp1m * __fact1
- (__ll + __m - 1) * __y_mm * __fact2) / _Tp(__ll - __m);
__y_mm = __y_mp1m;
__y_mp1m = __y_lm;
}
return __y_lm;
}
}
}
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __detail
{
template <typename _Tp>
void
__bessel_ik(_Tp __nu, _Tp __x,
_Tp & __Inu, _Tp & __Knu, _Tp & __Ipnu, _Tp & __Kpnu)
{
if (__x == _Tp(0))
{
if (__nu == _Tp(0))
{
__Inu = _Tp(1);
__Ipnu = _Tp(0);
}
else if (__nu == _Tp(1))
{
__Inu = _Tp(0);
__Ipnu = _Tp(0.5L);
}
else
{
__Inu = _Tp(0);
__Ipnu = _Tp(0);
}
__Knu = std::numeric_limits<_Tp>::infinity();
__Kpnu = -std::numeric_limits<_Tp>::infinity();
return;
}
const _Tp __eps = std::numeric_limits<_Tp>::epsilon();
const _Tp __fp_min = _Tp(10) * std::numeric_limits<_Tp>::epsilon();
const int __max_iter = 15000;
const _Tp __x_min = _Tp(2);
const int __nl = static_cast<int>(__nu + _Tp(0.5L));
const _Tp __mu = __nu - __nl;
const _Tp __mu2 = __mu * __mu;
const _Tp __xi = _Tp(1) / __x;
const _Tp __xi2 = _Tp(2) * __xi;
_Tp __h = __nu * __xi;
if ( __h < __fp_min )
__h = __fp_min;
_Tp __b = __xi2 * __nu;
_Tp __d = _Tp(0);
_Tp __c = __h;
int __i;
for ( __i = 1; __i <= __max_iter; ++__i )
{
__b += __xi2;
__d = _Tp(1) / (__b + __d);
__c = __b + _Tp(1) / __c;
const _Tp __del = __c * __d;
__h *= __del;
if (std::abs(__del - _Tp(1)) < __eps)
break;
}
if (__i > __max_iter)
std::__throw_runtime_error(("Argument x too large " "in __bessel_ik; " "try asymptotic expansion.")
);
_Tp __Inul = __fp_min;
_Tp __Ipnul = __h * __Inul;
_Tp __Inul1 = __Inul;
_Tp __Ipnu1 = __Ipnul;
_Tp __fact = __nu * __xi;
for (int __l = __nl; __l >= 1; --__l)
{
const _Tp __Inutemp = __fact * __Inul + __Ipnul;
__fact -= __xi;
__Ipnul = __fact * __Inutemp + __Inul;
__Inul = __Inutemp;
}
_Tp __f = __Ipnul / __Inul;
_Tp __Kmu, __Knu1;
if (__x < __x_min)
{
const _Tp __x2 = __x / _Tp(2);
const _Tp __pimu = __numeric_constants<_Tp>::__pi() * __mu;
const _Tp __fact = (std::abs(__pimu) < __eps
? _Tp(1) : __pimu / std::sin(__pimu));
_Tp __d = -std::log(__x2);
_Tp __e = __mu * __d;
const _Tp __fact2 = (std::abs(__e) < __eps
? _Tp(1) : std::sinh(__e) / __e);
_Tp __gam1, __gam2, __gampl, __gammi;
__gamma_temme(__mu, __gam1, __gam2, __gampl, __gammi);
_Tp __ff = __fact
* (__gam1 * std::cosh(__e) + __gam2 * __fact2 * __d);
_Tp __sum = __ff;
__e = std::exp(__e);
_Tp __p = __e / (_Tp(2) * __gampl);
_Tp __q = _Tp(1) / (_Tp(2) * __e * __gammi);
_Tp __c = _Tp(1);
__d = __x2 * __x2;
_Tp __sum1 = __p;
int __i;
for (__i = 1; __i <= __max_iter; ++__i)
{
__ff = (__i * __ff + __p + __q) / (__i * __i - __mu2);
__c *= __d / __i;
__p /= __i - __mu;
__q /= __i + __mu;
const _Tp __del = __c * __ff;
__sum += __del;
const _Tp __del1 = __c * (__p - __i * __ff);
__sum1 += __del1;
if (std::abs(__del) < __eps * std::abs(__sum))
break;
}
if (__i > __max_iter)
std::__throw_runtime_error(("Bessel k series failed to converge " "in __bessel_ik.")
);
__Kmu = __sum;
__Knu1 = __sum1 * __xi2;
}
else
{
_Tp __b = _Tp(2) * (_Tp(1) + __x);
_Tp __d = _Tp(1) / __b;
_Tp __delh = __d;
_Tp __h = __delh;
_Tp __q1 = _Tp(0);
_Tp __q2 = _Tp(1);
_Tp __a1 = _Tp(0.25L) - __mu2;
_Tp __q = __c = __a1;
_Tp __a = -__a1;
_Tp __s = _Tp(1) + __q * __delh;
int __i;
for (__i = 2; __i <= __max_iter; ++__i)
{
__a -= 2 * (__i - 1);
__c = -__a * __c / __i;
const _Tp __qnew = (__q1 - __b * __q2) / __a;
__q1 = __q2;
__q2 = __qnew;
__q += __c * __qnew;
__b += _Tp(2);
__d = _Tp(1) / (__b + __a * __d);
__delh = (__b * __d - _Tp(1)) * __delh;
__h += __delh;
const _Tp __dels = __q * __delh;
__s += __dels;
if ( std::abs(__dels / __s) < __eps )
break;
}
if (__i > __max_iter)
std::__throw_runtime_error(("Steed's method failed " "in __bessel_ik.")
);
__h = __a1 * __h;
__Kmu = std::sqrt(__numeric_constants<_Tp>::__pi() / (_Tp(2) * __x))
* std::exp(-__x) / __s;
__Knu1 = __Kmu * (__mu + __x + _Tp(0.5L) - __h) * __xi;
}
_Tp __Kpmu = __mu * __xi * __Kmu - __Knu1;
_Tp __Inumu = __xi / (__f * __Kmu - __Kpmu);
__Inu = __Inumu * __Inul1 / __Inul;
__Ipnu = __Inumu * __Ipnu1 / __Inul;
for ( __i = 1; __i <= __nl; ++__i )
{
const _Tp __Knutemp = (__mu + __i) * __xi2 * __Knu1 + __Kmu;
__Kmu = __Knu1;
__Knu1 = __Knutemp;
}
__Knu = __Kmu;
__Kpnu = __nu * __xi * __Kmu - __Knu1;
return;
}
template<typename _Tp>
_Tp
__cyl_bessel_i(_Tp __nu, _Tp __x)
{
if (__nu < _Tp(0) || __x < _Tp(0))
std::__throw_domain_error(("Bad argument " "in __cyl_bessel_i.")
);
else if (__isnan(__nu) || __isnan(__x))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (__x * __x < _Tp(10) * (__nu + _Tp(1)))
return __cyl_bessel_ij_series(__nu, __x, +_Tp(1), 200);
else
{
_Tp __I_nu, __K_nu, __Ip_nu, __Kp_nu;
__bessel_ik(__nu, __x, __I_nu, __K_nu, __Ip_nu, __Kp_nu);
return __I_nu;
}
}
template<typename _Tp>
_Tp
__cyl_bessel_k(_Tp __nu, _Tp __x)
{
if (__nu < _Tp(0) || __x < _Tp(0))
std::__throw_domain_error(("Bad argument " "in __cyl_bessel_k.")
);
else if (__isnan(__nu) || __isnan(__x))
return std::numeric_limits<_Tp>::quiet_NaN();
else
{
_Tp __I_nu, __K_nu, __Ip_nu, __Kp_nu;
__bessel_ik(__nu, __x, __I_nu, __K_nu, __Ip_nu, __Kp_nu);
return __K_nu;
}
}
template <typename _Tp>
void
__sph_bessel_ik(unsigned int __n, _Tp __x,
_Tp & __i_n, _Tp & __k_n, _Tp & __ip_n, _Tp & __kp_n)
{
const _Tp __nu = _Tp(__n) + _Tp(0.5L);
_Tp __I_nu, __Ip_nu, __K_nu, __Kp_nu;
__bessel_ik(__nu, __x, __I_nu, __K_nu, __Ip_nu, __Kp_nu);
const _Tp __factor = __numeric_constants<_Tp>::__sqrtpio2()
/ std::sqrt(__x);
__i_n = __factor * __I_nu;
__k_n = __factor * __K_nu;
__ip_n = __factor * __Ip_nu - __i_n / (_Tp(2) * __x);
__kp_n = __factor * __Kp_nu - __k_n / (_Tp(2) * __x);
return;
}
template <typename _Tp>
void
__airy(_Tp __x, _Tp & __Ai, _Tp & __Bi, _Tp & __Aip, _Tp & __Bip)
{
const _Tp __absx = std::abs(__x);
const _Tp __rootx = std::sqrt(__absx);
const _Tp __z = _Tp(2) * __absx * __rootx / _Tp(3);
const _Tp _S_inf = std::numeric_limits<_Tp>::infinity();
if (__isnan(__x))
__Bip = __Aip = __Bi = __Ai = std::numeric_limits<_Tp>::quiet_NaN();
else if (__z == _S_inf)
{
__Aip = __Ai = _Tp(0);
__Bip = __Bi = _S_inf;
}
else if (__z == -_S_inf)
__Bip = __Aip = __Bi = __Ai = _Tp(0);
else if (__x > _Tp(0))
{
_Tp __I_nu, __Ip_nu, __K_nu, __Kp_nu;
__bessel_ik(_Tp(1) / _Tp(3), __z, __I_nu, __K_nu, __Ip_nu, __Kp_nu);
__Ai = __rootx * __K_nu
/ (__numeric_constants<_Tp>::__sqrt3()
* __numeric_constants<_Tp>::__pi());
__Bi = __rootx * (__K_nu / __numeric_constants<_Tp>::__pi()
+ _Tp(2) * __I_nu / __numeric_constants<_Tp>::__sqrt3());
__bessel_ik(_Tp(2) / _Tp(3), __z, __I_nu, __K_nu, __Ip_nu, __Kp_nu);
__Aip = -__x * __K_nu
/ (__numeric_constants<_Tp>::__sqrt3()
* __numeric_constants<_Tp>::__pi());
__Bip = __x * (__K_nu / __numeric_constants<_Tp>::__pi()
+ _Tp(2) * __I_nu
/ __numeric_constants<_Tp>::__sqrt3());
}
else if (__x < _Tp(0))
{
_Tp __J_nu, __Jp_nu, __N_nu, __Np_nu;
__bessel_jn(_Tp(1) / _Tp(3), __z, __J_nu, __N_nu, __Jp_nu, __Np_nu);
__Ai = __rootx * (__J_nu
- __N_nu / __numeric_constants<_Tp>::__sqrt3()) / _Tp(2);
__Bi = -__rootx * (__N_nu
+ __J_nu / __numeric_constants<_Tp>::__sqrt3()) / _Tp(2);
__bessel_jn(_Tp(2) / _Tp(3), __z, __J_nu, __N_nu, __Jp_nu, __Np_nu);
__Aip = __absx * (__N_nu / __numeric_constants<_Tp>::__sqrt3()
+ __J_nu) / _Tp(2);
__Bip = __absx * (__J_nu / __numeric_constants<_Tp>::__sqrt3()
- __N_nu) / _Tp(2);
}
else
{
__Ai = _Tp(0.35502805388781723926L);
__Bi = __Ai * __numeric_constants<_Tp>::__sqrt3();
__Aip = -_Tp(0.25881940379280679840L);
__Bip = -__Aip * __numeric_constants<_Tp>::__sqrt3();
}
return;
}
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __detail
{
template<typename _Tp>
_Tp
__poly_hermite_recursion(unsigned int __n, _Tp __x)
{
_Tp __H_0 = 1;
if (__n == 0)
return __H_0;
_Tp __H_1 = 2 * __x;
if (__n == 1)
return __H_1;
_Tp __H_n, __H_nm1, __H_nm2;
unsigned int __i;
for (__H_nm2 = __H_0, __H_nm1 = __H_1, __i = 2; __i <= __n; ++__i)
{
__H_n = 2 * (__x * __H_nm1 - (__i - 1) * __H_nm2);
__H_nm2 = __H_nm1;
__H_nm1 = __H_n;
}
return __H_n;
}
template<typename _Tp>
inline _Tp
__poly_hermite(unsigned int __n, _Tp __x)
{
if (__isnan(__x))
return std::numeric_limits<_Tp>::quiet_NaN();
else
return __poly_hermite_recursion(__n, __x);
}
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __detail
{
template<typename _Tpa, typename _Tp>
_Tp
__poly_laguerre_large_n(unsigned __n, _Tpa __alpha1, _Tp __x)
{
const _Tp __a = -_Tp(__n);
const _Tp __b = _Tp(__alpha1) + _Tp(1);
const _Tp __eta = _Tp(2) * __b - _Tp(4) * __a;
const _Tp __cos2th = __x / __eta;
const _Tp __sin2th = _Tp(1) - __cos2th;
const _Tp __th = std::acos(std::sqrt(__cos2th));
const _Tp __pre_h = __numeric_constants<_Tp>::__pi_2()
* __numeric_constants<_Tp>::__pi_2()
* __eta * __eta * __cos2th * __sin2th;
const _Tp __lg_b = ::std::lgamma(_Tp(__n) + __b);
const _Tp __lnfact = ::std::lgamma(_Tp(__n + 1));
_Tp __pre_term1 = _Tp(0.5L) * (_Tp(1) - __b)
* std::log(_Tp(0.25L) * __x * __eta);
_Tp __pre_term2 = _Tp(0.25L) * std::log(__pre_h);
_Tp __lnpre = __lg_b - __lnfact + _Tp(0.5L) * __x
+ __pre_term1 - __pre_term2;
_Tp __ser_term1 = std::sin(__a * __numeric_constants<_Tp>::__pi());
_Tp __ser_term2 = std::sin(_Tp(0.25L) * __eta
* (_Tp(2) * __th
- std::sin(_Tp(2) * __th))
+ __numeric_constants<_Tp>::__pi_4());
_Tp __ser = __ser_term1 + __ser_term2;
return std::exp(__lnpre) * __ser;
}
template<typename _Tpa, typename _Tp>
_Tp
__poly_laguerre_hyperg(unsigned int __n, _Tpa __alpha1, _Tp __x)
{
const _Tp __b = _Tp(__alpha1) + _Tp(1);
const _Tp __mx = -__x;
const _Tp __tc_sgn = (__x < _Tp(0) ? _Tp(1)
: ((__n % 2 == 1) ? -_Tp(1) : _Tp(1)));
_Tp __tc = _Tp(1);
const _Tp __ax = std::abs(__x);
for (unsigned int __k = 1; __k <= __n; ++__k)
__tc *= (__ax / __k);
_Tp __term = __tc * __tc_sgn;
_Tp __sum = __term;
for (int __k = int(__n) - 1; __k >= 0; --__k)
{
__term *= ((__b + _Tp(__k)) / _Tp(int(__n) - __k))
* _Tp(__k + 1) / __mx;
__sum += __term;
}
return __sum;
}
template<typename _Tpa, typename _Tp>
_Tp
__poly_laguerre_recursion(unsigned int __n, _Tpa __alpha1, _Tp __x)
{
_Tp __l_0 = _Tp(1);
if (__n == 0)
return __l_0;
_Tp __l_1 = -__x + _Tp(1) + _Tp(__alpha1);
if (__n == 1)
return __l_1;
_Tp __l_n2 = __l_0;
_Tp __l_n1 = __l_1;
_Tp __l_n = _Tp(0);
for (unsigned int __nn = 2; __nn <= __n; ++__nn)
{
__l_n = (_Tp(2 * __nn - 1) + _Tp(__alpha1) - __x)
* __l_n1 / _Tp(__nn)
- (_Tp(__nn - 1) + _Tp(__alpha1)) * __l_n2 / _Tp(__nn);
__l_n2 = __l_n1;
__l_n1 = __l_n;
}
return __l_n;
}
template<typename _Tpa, typename _Tp>
_Tp
__poly_laguerre(unsigned int __n, _Tpa __alpha1, _Tp __x)
{
if (__x < _Tp(0))
std::__throw_domain_error(("Negative argument " "in __poly_laguerre.")
);
else if (__isnan(__x))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (__n == 0)
return _Tp(1);
else if (__n == 1)
return _Tp(1) + _Tp(__alpha1) - __x;
else if (__x == _Tp(0))
{
_Tp __prod = _Tp(__alpha1) + _Tp(1);
for (unsigned int __k = 2; __k <= __n; ++__k)
__prod *= (_Tp(__alpha1) + _Tp(__k)) / _Tp(__k);
return __prod;
}
else if (__n > 10000000 && _Tp(__alpha1) > -_Tp(1)
&& __x < _Tp(2) * (_Tp(__alpha1) + _Tp(1)) + _Tp(4 * __n))
return __poly_laguerre_large_n(__n, __alpha1, __x);
else if (_Tp(__alpha1) >= _Tp(0)
|| (__x > _Tp(0) && _Tp(__alpha1) < -_Tp(__n + 1)))
return __poly_laguerre_recursion(__n, __alpha1, __x);
else
return __poly_laguerre_hyperg(__n, __alpha1, __x);
}
template<typename _Tp>
inline _Tp
__assoc_laguerre(unsigned int __n, unsigned int __m, _Tp __x)
{ return __poly_laguerre<unsigned int, _Tp>(__n, __m, __x); }
template<typename _Tp>
inline _Tp
__laguerre(unsigned int __n, _Tp __x)
{ return __poly_laguerre<unsigned int, _Tp>(__n, 0, __x); }
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __detail
{
template<typename _Tp>
_Tp
__riemann_zeta_sum(_Tp __s)
{
if (__s < _Tp(1))
std::__throw_domain_error(("Bad argument in zeta sum."));
const unsigned int max_iter = 10000;
_Tp __zeta = _Tp(0);
for (unsigned int __k = 1; __k < max_iter; ++__k)
{
_Tp __term = std::pow(static_cast<_Tp>(__k), -__s);
if (__term < std::numeric_limits<_Tp>::epsilon())
{
break;
}
__zeta += __term;
}
return __zeta;
}
template<typename _Tp>
_Tp
__riemann_zeta_alt(_Tp __s)
{
_Tp __sgn = _Tp(1);
_Tp __zeta = _Tp(0);
for (unsigned int __i = 1; __i < 10000000; ++__i)
{
_Tp __term = __sgn / std::pow(__i, __s);
if (std::abs(__term) < std::numeric_limits<_Tp>::epsilon())
break;
__zeta += __term;
__sgn *= _Tp(-1);
}
__zeta /= _Tp(1) - std::pow(_Tp(2), _Tp(1) - __s);
return __zeta;
}
template<typename _Tp>
_Tp
__riemann_zeta_glob(_Tp __s)
{
_Tp __zeta = _Tp(0);
const _Tp __eps = std::numeric_limits<_Tp>::epsilon();
const _Tp __max_bincoeff = std::numeric_limits<_Tp>::max_exponent10
* std::log(_Tp(10)) - _Tp(1);
if (__s < _Tp(0))
{
if (::std::fmod(__s,_Tp(2)) == _Tp(0))
return _Tp(0);
else
{
_Tp __zeta = __riemann_zeta_glob(_Tp(1) - __s);
__zeta *= std::pow(_Tp(2)
* __numeric_constants<_Tp>::__pi(), __s)
* std::sin(__numeric_constants<_Tp>::__pi_2() * __s)
* std::exp(::std::lgamma(_Tp(1) - __s))
/ __numeric_constants<_Tp>::__pi();
return __zeta;
}
}
_Tp __num = _Tp(0.5L);
const unsigned int __maxit = 10000;
for (unsigned int __i = 0; __i < __maxit; ++__i)
{
bool __punt = false;
_Tp __sgn = _Tp(1);
_Tp __term = _Tp(0);
for (unsigned int __j = 0; __j <= __i; ++__j)
{
_Tp __bincoeff = ::std::lgamma(_Tp(1 + __i))
- ::std::lgamma(_Tp(1 + __j))
- ::std::lgamma(_Tp(1 + __i - __j));
if (__bincoeff > __max_bincoeff)
{
__punt = true;
break;
}
__bincoeff = std::exp(__bincoeff);
__term += __sgn * __bincoeff * std::pow(_Tp(1 + __j), -__s);
__sgn *= _Tp(-1);
}
if (__punt)
break;
__term *= __num;
__zeta += __term;
if (std::abs(__term/__zeta) < __eps)
break;
__num *= _Tp(0.5L);
}
__zeta /= _Tp(1) - std::pow(_Tp(2), _Tp(1) - __s);
return __zeta;
}
template<typename _Tp>
_Tp
__riemann_zeta_product(_Tp __s)
{
static const _Tp __prime[] = {
_Tp(2), _Tp(3), _Tp(5), _Tp(7), _Tp(11), _Tp(13), _Tp(17), _Tp(19),
_Tp(23), _Tp(29), _Tp(31), _Tp(37), _Tp(41), _Tp(43), _Tp(47),
_Tp(53), _Tp(59), _Tp(61), _Tp(67), _Tp(71), _Tp(73), _Tp(79),
_Tp(83), _Tp(89), _Tp(97), _Tp(101), _Tp(103), _Tp(107), _Tp(109)
};
static const unsigned int __num_primes = sizeof(__prime) / sizeof(_Tp);
_Tp __zeta = _Tp(1);
for (unsigned int __i = 0; __i < __num_primes; ++__i)
{
const _Tp __fact = _Tp(1) - std::pow(__prime[__i], -__s);
__zeta *= __fact;
if (_Tp(1) - __fact < std::numeric_limits<_Tp>::epsilon())
break;
}
__zeta = _Tp(1) / __zeta;
return __zeta;
}
template<typename _Tp>
_Tp
__riemann_zeta(_Tp __s)
{
if (__isnan(__s))
return std::numeric_limits<_Tp>::quiet_NaN();
else if (__s == _Tp(1))
return std::numeric_limits<_Tp>::infinity();
else if (__s < -_Tp(19))
{
_Tp __zeta = __riemann_zeta_product(_Tp(1) - __s);
__zeta *= std::pow(_Tp(2) * __numeric_constants<_Tp>::__pi(), __s)
* std::sin(__numeric_constants<_Tp>::__pi_2() * __s)
* std::exp(::std::lgamma(_Tp(1) - __s))
/ __numeric_constants<_Tp>::__pi();
return __zeta;
}
else if (__s < _Tp(20))
{
bool __glob = true;
if (__glob)
return __riemann_zeta_glob(__s);
else
{
if (__s > _Tp(1))
return __riemann_zeta_sum(__s);
else
{
_Tp __zeta = std::pow(_Tp(2)
* __numeric_constants<_Tp>::__pi(), __s)
* std::sin(__numeric_constants<_Tp>::__pi_2() * __s)
* ::std::tgamma(_Tp(1) - __s)
* __riemann_zeta_sum(_Tp(1) - __s);
return __zeta;
}
}
}
else
return __riemann_zeta_product(__s);
}
template<typename _Tp>
_Tp
__hurwitz_zeta_glob(_Tp __a, _Tp __s)
{
_Tp __zeta = _Tp(0);
const _Tp __eps = std::numeric_limits<_Tp>::epsilon();
const _Tp __max_bincoeff = std::numeric_limits<_Tp>::max_exponent10
* std::log(_Tp(10)) - _Tp(1);
const unsigned int __maxit = 10000;
for (unsigned int __i = 0; __i < __maxit; ++__i)
{
bool __punt = false;
_Tp __sgn = _Tp(1);
_Tp __term = _Tp(0);
for (unsigned int __j = 0; __j <= __i; ++__j)
{
_Tp __bincoeff = ::std::lgamma(_Tp(1 + __i))
- ::std::lgamma(_Tp(1 + __j))
- ::std::lgamma(_Tp(1 + __i - __j));
if (__bincoeff > __max_bincoeff)
{
__punt = true;
break;
}
__bincoeff = std::exp(__bincoeff);
__term += __sgn * __bincoeff * std::pow(_Tp(__a + __j), -__s);
__sgn *= _Tp(-1);
}
if (__punt)
break;
__term /= _Tp(__i + 1);
if (std::abs(__term / __zeta) < __eps)
break;
__zeta += __term;
}
__zeta /= __s - _Tp(1);
return __zeta;
}
template<typename _Tp>
inline _Tp
__hurwitz_zeta(_Tp __a, _Tp __s)
{ return __hurwitz_zeta_glob(__a, __s); }
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
inline float
assoc_laguerref(unsigned int __n, unsigned int __m, float __x)
{ return __detail::__assoc_laguerre<float>(__n, __m, __x); }
inline long double
assoc_laguerrel(unsigned int __n, unsigned int __m, long double __x)
{ return __detail::__assoc_laguerre<long double>(__n, __m, __x); }
template<typename _Tp>
inline typename __gnu_cxx::__promote<_Tp>::__type
assoc_laguerre(unsigned int __n, unsigned int __m, _Tp __x)
{
typedef typename __gnu_cxx::__promote<_Tp>::__type __type;
return __detail::__assoc_laguerre<__type>(__n, __m, __x);
}
inline float
assoc_legendref(unsigned int __l, unsigned int __m, float __x)
{ return __detail::__assoc_legendre_p<float>(__l, __m, __x); }
inline long double
assoc_legendrel(unsigned int __l, unsigned int __m, long double __x)
{ return __detail::__assoc_legendre_p<long double>(__l, __m, __x); }
template<typename _Tp>
inline typename __gnu_cxx::__promote<_Tp>::__type
assoc_legendre(unsigned int __l, unsigned int __m, _Tp __x)
{
typedef typename __gnu_cxx::__promote<_Tp>::__type __type;
return __detail::__assoc_legendre_p<__type>(__l, __m, __x);
}
inline float
betaf(float __a, float __b)
{ return __detail::__beta<float>(__a, __b); }
inline long double
betal(long double __a, long double __b)
{ return __detail::__beta<long double>(__a, __b); }
template<typename _Tpa, typename _Tpb>
inline typename __gnu_cxx::__promote_2<_Tpa, _Tpb>::__type
beta(_Tpa __a, _Tpb __b)
{
typedef typename __gnu_cxx::__promote_2<_Tpa, _Tpb>::__type __type;
return __detail::__beta<__type>(__a, __b);
}
inline float
comp_ellint_1f(float __k)
{ return __detail::__comp_ellint_1<float>(__k); }
inline long double
comp_ellint_1l(long double __k)
{ return __detail::__comp_ellint_1<long double>(__k); }
template<typename _Tp>
inline typename __gnu_cxx::__promote<_Tp>::__type
comp_ellint_1(_Tp __k)
{
typedef typename __gnu_cxx::__promote<_Tp>::__type __type;
return __detail::__comp_ellint_1<__type>(__k);
}
inline float
comp_ellint_2f(float __k)
{ return __detail::__comp_ellint_2<float>(__k); }
inline long double
comp_ellint_2l(long double __k)
{ return __detail::__comp_ellint_2<long double>(__k); }
template<typename _Tp>
inline typename __gnu_cxx::__promote<_Tp>::__type
comp_ellint_2(_Tp __k)
{
typedef typename __gnu_cxx::__promote<_Tp>::__type __type;
return __detail::__comp_ellint_2<__type>(__k);
}
inline float
comp_ellint_3f(float __k, float __nu)
{ return __detail::__comp_ellint_3<float>(__k, __nu); }
inline long double
comp_ellint_3l(long double __k, long double __nu)
{ return __detail::__comp_ellint_3<long double>(__k, __nu); }
template<typename _Tp, typename _Tpn>
inline typename __gnu_cxx::__promote_2<_Tp, _Tpn>::__type
comp_ellint_3(_Tp __k, _Tpn __nu)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Tpn>::__type __type;
return __detail::__comp_ellint_3<__type>(__k, __nu);
}
inline float
cyl_bessel_if(float __nu, float __x)
{ return __detail::__cyl_bessel_i<float>(__nu, __x); }
inline long double
cyl_bessel_il(long double __nu, long double __x)
{ return __detail::__cyl_bessel_i<long double>(__nu, __x); }
template<typename _Tpnu, typename _Tp>
inline typename __gnu_cxx::__promote_2<_Tpnu, _Tp>::__type
cyl_bessel_i(_Tpnu __nu, _Tp __x)
{
typedef typename __gnu_cxx::__promote_2<_Tpnu, _Tp>::__type __type;
return __detail::__cyl_bessel_i<__type>(__nu, __x);
}
inline float
cyl_bessel_jf(float __nu, float __x)
{ return __detail::__cyl_bessel_j<float>(__nu, __x); }
inline long double
cyl_bessel_jl(long double __nu, long double __x)
{ return __detail::__cyl_bessel_j<long double>(__nu, __x); }
template<typename _Tpnu, typename _Tp>
inline typename __gnu_cxx::__promote_2<_Tpnu, _Tp>::__type
cyl_bessel_j(_Tpnu __nu, _Tp __x)
{
typedef typename __gnu_cxx::__promote_2<_Tpnu, _Tp>::__type __type;
return __detail::__cyl_bessel_j<__type>(__nu, __x);
}
inline float
cyl_bessel_kf(float __nu, float __x)
{ return __detail::__cyl_bessel_k<float>(__nu, __x); }
inline long double
cyl_bessel_kl(long double __nu, long double __x)
{ return __detail::__cyl_bessel_k<long double>(__nu, __x); }
template<typename _Tpnu, typename _Tp>
inline typename __gnu_cxx::__promote_2<_Tpnu, _Tp>::__type
cyl_bessel_k(_Tpnu __nu, _Tp __x)
{
typedef typename __gnu_cxx::__promote_2<_Tpnu, _Tp>::__type __type;
return __detail::__cyl_bessel_k<__type>(__nu, __x);
}
inline float
cyl_neumannf(float __nu, float __x)
{ return __detail::__cyl_neumann_n<float>(__nu, __x); }
inline long double
cyl_neumannl(long double __nu, long double __x)
{ return __detail::__cyl_neumann_n<long double>(__nu, __x); }
template<typename _Tpnu, typename _Tp>
inline typename __gnu_cxx::__promote_2<_Tpnu, _Tp>::__type
cyl_neumann(_Tpnu __nu, _Tp __x)
{
typedef typename __gnu_cxx::__promote_2<_Tpnu, _Tp>::__type __type;
return __detail::__cyl_neumann_n<__type>(__nu, __x);
}
inline float
ellint_1f(float __k, float __phi)
{ return __detail::__ellint_1<float>(__k, __phi); }
inline long double
ellint_1l(long double __k, long double __phi)
{ return __detail::__ellint_1<long double>(__k, __phi); }
template<typename _Tp, typename _Tpp>
inline typename __gnu_cxx::__promote_2<_Tp, _Tpp>::__type
ellint_1(_Tp __k, _Tpp __phi)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Tpp>::__type __type;
return __detail::__ellint_1<__type>(__k, __phi);
}
inline float
ellint_2f(float __k, float __phi)
{ return __detail::__ellint_2<float>(__k, __phi); }
inline long double
ellint_2l(long double __k, long double __phi)
{ return __detail::__ellint_2<long double>(__k, __phi); }
template<typename _Tp, typename _Tpp>
inline typename __gnu_cxx::__promote_2<_Tp, _Tpp>::__type
ellint_2(_Tp __k, _Tpp __phi)
{
typedef typename __gnu_cxx::__promote_2<_Tp, _Tpp>::__type __type;
return __detail::__ellint_2<__type>(__k, __phi);
}
inline float
ellint_3f(float __k, float __nu, float __phi)
{ return __detail::__ellint_3<float>(__k, __nu, __phi); }
inline long double
ellint_3l(long double __k, long double __nu, long double __phi)
{ return __detail::__ellint_3<long double>(__k, __nu, __phi); }
template<typename _Tp, typename _Tpn, typename _Tpp>
inline typename __gnu_cxx::__promote_3<_Tp, _Tpn, _Tpp>::__type
ellint_3(_Tp __k, _Tpn __nu, _Tpp __phi)
{
typedef typename __gnu_cxx::__promote_3<_Tp, _Tpn, _Tpp>::__type __type;
return __detail::__ellint_3<__type>(__k, __nu, __phi);
}
inline float
expintf(float __x)
{ return __detail::__expint<float>(__x); }
inline long double
expintl(long double __x)
{ return __detail::__expint<long double>(__x); }
template<typename _Tp>
inline typename __gnu_cxx::__promote<_Tp>::__type
expint(_Tp __x)
{
typedef typename __gnu_cxx::__promote<_Tp>::__type __type;
return __detail::__expint<__type>(__x);
}
inline float
hermitef(unsigned int __n, float __x)
{ return __detail::__poly_hermite<float>(__n, __x); }
inline long double
hermitel(unsigned int __n, long double __x)
{ return __detail::__poly_hermite<long double>(__n, __x); }
template<typename _Tp>
inline typename __gnu_cxx::__promote<_Tp>::__type
hermite(unsigned int __n, _Tp __x)
{
typedef typename __gnu_cxx::__promote<_Tp>::__type __type;
return __detail::__poly_hermite<__type>(__n, __x);
}
inline float
laguerref(unsigned int __n, float __x)
{ return __detail::__laguerre<float>(__n, __x); }
inline long double
laguerrel(unsigned int __n, long double __x)
{ return __detail::__laguerre<long double>(__n, __x); }
template<typename _Tp>
inline typename __gnu_cxx::__promote<_Tp>::__type
laguerre(unsigned int __n, _Tp __x)
{
typedef typename __gnu_cxx::__promote<_Tp>::__type __type;
return __detail::__laguerre<__type>(__n, __x);
}
inline float
legendref(unsigned int __l, float __x)
{ return __detail::__poly_legendre_p<float>(__l, __x); }
inline long double
legendrel(unsigned int __l, long double __x)
{ return __detail::__poly_legendre_p<long double>(__l, __x); }
template<typename _Tp>
inline typename __gnu_cxx::__promote<_Tp>::__type
legendre(unsigned int __l, _Tp __x)
{
typedef typename __gnu_cxx::__promote<_Tp>::__type __type;
return __detail::__poly_legendre_p<__type>(__l, __x);
}
inline float
riemann_zetaf(float __s)
{ return __detail::__riemann_zeta<float>(__s); }
inline long double
riemann_zetal(long double __s)
{ return __detail::__riemann_zeta<long double>(__s); }
template<typename _Tp>
inline typename __gnu_cxx::__promote<_Tp>::__type
riemann_zeta(_Tp __s)
{
typedef typename __gnu_cxx::__promote<_Tp>::__type __type;
return __detail::__riemann_zeta<__type>(__s);
}
inline float
sph_besself(unsigned int __n, float __x)
{ return __detail::__sph_bessel<float>(__n, __x); }
inline long double
sph_bessell(unsigned int __n, long double __x)
{ return __detail::__sph_bessel<long double>(__n, __x); }
template<typename _Tp>
inline typename __gnu_cxx::__promote<_Tp>::__type
sph_bessel(unsigned int __n, _Tp __x)
{
typedef typename __gnu_cxx::__promote<_Tp>::__type __type;
return __detail::__sph_bessel<__type>(__n, __x);
}
inline float
sph_legendref(unsigned int __l, unsigned int __m, float __theta)
{ return __detail::__sph_legendre<float>(__l, __m, __theta); }
inline long double
sph_legendrel(unsigned int __l, unsigned int __m, long double __theta)
{ return __detail::__sph_legendre<long double>(__l, __m, __theta); }
template<typename _Tp>
inline typename __gnu_cxx::__promote<_Tp>::__type
sph_legendre(unsigned int __l, unsigned int __m, _Tp __theta)
{
typedef typename __gnu_cxx::__promote<_Tp>::__type __type;
return __detail::__sph_legendre<__type>(__l, __m, __theta);
}
inline float
sph_neumannf(unsigned int __n, float __x)
{ return __detail::__sph_neumann<float>(__n, __x); }
inline long double
sph_neumannl(unsigned int __n, long double __x)
{ return __detail::__sph_neumann<long double>(__n, __x); }
template<typename _Tp>
inline typename __gnu_cxx::__promote<_Tp>::__type
sph_neumann(unsigned int __n, _Tp __x)
{
typedef typename __gnu_cxx::__promote<_Tp>::__type __type;
return __detail::__sph_neumann<__type>(__n, __x);
}
}
namespace __gnu_cxx __attribute__ ((__visibility__ ("default")))
{
inline float
airy_aif(float __x)
{
float __Ai, __Bi, __Aip, __Bip;
std::__detail::__airy<float>(__x, __Ai, __Bi, __Aip, __Bip);
return __Ai;
}
inline long double
airy_ail(long double __x)
{
long double __Ai, __Bi, __Aip, __Bip;
std::__detail::__airy<long double>(__x, __Ai, __Bi, __Aip, __Bip);
return __Ai;
}
template<typename _Tp>
inline typename __gnu_cxx::__promote<_Tp>::__type
airy_ai(_Tp __x)
{
typedef typename __gnu_cxx::__promote<_Tp>::__type __type;
__type __Ai, __Bi, __Aip, __Bip;
std::__detail::__airy<__type>(__x, __Ai, __Bi, __Aip, __Bip);
return __Ai;
}
inline float
airy_bif(float __x)
{
float __Ai, __Bi, __Aip, __Bip;
std::__detail::__airy<float>(__x, __Ai, __Bi, __Aip, __Bip);
return __Bi;
}
inline long double
airy_bil(long double __x)
{
long double __Ai, __Bi, __Aip, __Bip;
std::__detail::__airy<long double>(__x, __Ai, __Bi, __Aip, __Bip);
return __Bi;
}
template<typename _Tp>
inline typename __gnu_cxx::__promote<_Tp>::__type
airy_bi(_Tp __x)
{
typedef typename __gnu_cxx::__promote<_Tp>::__type __type;
__type __Ai, __Bi, __Aip, __Bip;
std::__detail::__airy<__type>(__x, __Ai, __Bi, __Aip, __Bip);
return __Bi;
}
inline float
conf_hypergf(float __a, float __c, float __x)
{ return std::__detail::__conf_hyperg<float>(__a, __c, __x); }
inline long double
conf_hypergl(long double __a, long double __c, long double __x)
{ return std::__detail::__conf_hyperg<long double>(__a, __c, __x); }
template<typename _Tpa, typename _Tpc, typename _Tp>
inline typename __gnu_cxx::__promote_3<_Tpa, _Tpc, _Tp>::__type
conf_hyperg(_Tpa __a, _Tpc __c, _Tp __x)
{
typedef typename __gnu_cxx::__promote_3<_Tpa, _Tpc, _Tp>::__type __type;
return std::__detail::__conf_hyperg<__type>(__a, __c, __x);
}
inline float
hypergf(float __a, float __b, float __c, float __x)
{ return std::__detail::__hyperg<float>(__a, __b, __c, __x); }
inline long double
hypergl(long double __a, long double __b, long double __c, long double __x)
{ return std::__detail::__hyperg<long double>(__a, __b, __c, __x); }
template<typename _Tpa, typename _Tpb, typename _Tpc, typename _Tp>
inline typename __gnu_cxx::__promote_4<_Tpa, _Tpb, _Tpc, _Tp>::__type
hyperg(_Tpa __a, _Tpb __b, _Tpc __c, _Tp __x)
{
typedef typename __gnu_cxx::__promote_4<_Tpa, _Tpb, _Tpc, _Tp>
::__type __type;
return std::__detail::__hyperg<__type>(__a, __b, __c, __x);
}
}
}
typedef int ptrdiff_t;
typedef struct {
long long __max_align_ll __attribute__((__aligned__(__alignof__(long long))));
long double __max_align_ld __attribute__((__aligned__(__alignof__(long double))));
__float128 __max_align_f128 __attribute__((__aligned__(__alignof(__float128))));
} max_align_t;
typedef decltype(nullptr) nullptr_t;
typedef __uint8_t uint8_t;
typedef __uint16_t uint16_t;
typedef __uint32_t uint32_t;
typedef __uint64_t uint64_t;
typedef __int_least8_t int_least8_t;
typedef __int_least16_t int_least16_t;
typedef __int_least32_t int_least32_t;
typedef __int_least64_t int_least64_t;
typedef __uint_least8_t uint_least8_t;
typedef __uint_least16_t uint_least16_t;
typedef __uint_least32_t uint_least32_t;
typedef __uint_least64_t uint_least64_t;
typedef signed char int_fast8_t;
typedef int int_fast16_t;
typedef int int_fast32_t;
__extension__
typedef long long int int_fast64_t;
typedef unsigned char uint_fast8_t;
typedef unsigned int uint_fast16_t;
typedef unsigned int uint_fast32_t;
__extension__
typedef unsigned long long int uint_fast64_t;
typedef int intptr_t;
typedef unsigned int uintptr_t;
typedef __intmax_t intmax_t;
typedef __uintmax_t uintmax_t;
extern "C" {
typedef struct
{
__extension__ long long int quot;
__extension__ long long int rem;
} imaxdiv_t;
extern intmax_t imaxabs (intmax_t __n) noexcept (true) __attribute__ ((__const__));
extern imaxdiv_t imaxdiv (intmax_t __numer, intmax_t __denom)
noexcept (true) __attribute__ ((__const__));
extern intmax_t strtoimax (const char *__restrict __nptr,
char **__restrict __endptr, int __base) noexcept (true);
extern uintmax_t strtoumax (const char *__restrict __nptr,
char ** __restrict __endptr, int __base) noexcept (true);
extern intmax_t wcstoimax (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr, int __base)
noexcept (true);
extern uintmax_t wcstoumax (const wchar_t *__restrict __nptr,
wchar_t ** __restrict __endptr, int __base)
noexcept (true);
}
extern "C++" {
namespace std __attribute__ ((__visibility__ ("default")))
{
class exception
{
public:
exception() noexcept { }
virtual ~exception() noexcept;
exception(const exception&) = default;
exception& operator=(const exception&) = default;
exception(exception&&) = default;
exception& operator=(exception&&) = default;
virtual const char*
what() const noexcept;
};
}
}
#pragma GCC visibility push(default)
extern "C++" {
namespace std
{
class bad_alloc : public exception
{
public:
bad_alloc() throw() { }
bad_alloc(const bad_alloc&) = default;
bad_alloc& operator=(const bad_alloc&) = default;
virtual ~bad_alloc() throw();
virtual const char* what() const throw();
};
class bad_array_new_length : public bad_alloc
{
public:
bad_array_new_length() throw() { }
virtual ~bad_array_new_length() throw();
virtual const char* what() const throw();
};
enum class align_val_t: size_t {};
struct nothrow_t
{
explicit nothrow_t() = default;
};
extern const nothrow_t nothrow;
typedef void (*new_handler)();
new_handler set_new_handler(new_handler) throw();
new_handler get_new_handler() noexcept;
}
[[__nodiscard__]] void* operator new(std::size_t)
__attribute__((__externally_visible__));
[[__nodiscard__]] void* operator new[](std::size_t)
__attribute__((__externally_visible__));
void operator delete(void*) noexcept
__attribute__((__externally_visible__));
void operator delete[](void*) noexcept
__attribute__((__externally_visible__));
void operator delete(void*, std::size_t) noexcept
__attribute__((__externally_visible__));
void operator delete[](void*, std::size_t) noexcept
__attribute__((__externally_visible__));
[[__nodiscard__]] void* operator new(std::size_t, const std::nothrow_t&) noexcept
__attribute__((__externally_visible__, __alloc_size__ (1), __malloc__));
[[__nodiscard__]] void* operator new[](std::size_t, const std::nothrow_t&) noexcept
__attribute__((__externally_visible__, __alloc_size__ (1), __malloc__));
void operator delete(void*, const std::nothrow_t&) noexcept
__attribute__((__externally_visible__));
void operator delete[](void*, const std::nothrow_t&) noexcept
__attribute__((__externally_visible__));
[[__nodiscard__]] void* operator new(std::size_t, std::align_val_t)
__attribute__((__externally_visible__, __alloc_size__ (1), __malloc__));
[[__nodiscard__]] void* operator new(std::size_t, std::align_val_t, const std::nothrow_t&)
noexcept __attribute__((__externally_visible__, __alloc_size__ (1), __malloc__));
void operator delete(void*, std::align_val_t)
noexcept __attribute__((__externally_visible__));
void operator delete(void*, std::align_val_t, const std::nothrow_t&)
noexcept __attribute__((__externally_visible__));
[[__nodiscard__]] void* operator new[](std::size_t, std::align_val_t)
__attribute__((__externally_visible__, __alloc_size__ (1), __malloc__));
[[__nodiscard__]] void* operator new[](std::size_t, std::align_val_t, const std::nothrow_t&)
noexcept __attribute__((__externally_visible__, __alloc_size__ (1), __malloc__));
void operator delete[](void*, std::align_val_t)
noexcept __attribute__((__externally_visible__));
void operator delete[](void*, std::align_val_t, const std::nothrow_t&)
noexcept __attribute__((__externally_visible__));
void operator delete(void*, std::size_t, std::align_val_t)
noexcept __attribute__((__externally_visible__));
void operator delete[](void*, std::size_t, std::align_val_t)
noexcept __attribute__((__externally_visible__));
[[__nodiscard__]] inline void* operator new(std::size_t, void* __p) noexcept
{ return __p; }
[[__nodiscard__]] inline void* operator new[](std::size_t, void* __p) noexcept
{ return __p; }
inline void operator delete (void*, void*) noexcept { }
inline void operator delete[](void*, void*) noexcept { }
}
namespace std
{
template<typename _Tp>
[[nodiscard]] constexpr _Tp*
launder(_Tp* __p) noexcept
{ return __builtin_launder(__p); }
template<typename _Ret, typename... _Args , bool _NE>
void launder(_Ret (*)(_Args...) noexcept (_NE)) = delete;
template<typename _Ret, typename... _Args , bool _NE>
void launder(_Ret (*)(_Args......) noexcept (_NE)) = delete;
void launder(void*) = delete;
void launder(const void*) = delete;
void launder(volatile void*) = delete;
void launder(const volatile void*) = delete;
inline constexpr size_t hardware_destructive_interference_size = 64;
inline constexpr size_t hardware_constructive_interference_size = 64;
}
#pragma GCC visibility pop
namespace std __attribute__ ((__visibility__ ("default")))
{
typedef enum memory_order
{
memory_order_relaxed,
memory_order_consume,
memory_order_acquire,
memory_order_release,
memory_order_acq_rel,
memory_order_seq_cst
} memory_order;
enum __memory_order_modifier
{
__memory_order_mask = 0x0ffff,
__memory_order_modifier_mask = 0xffff0000,
__memory_order_hle_acquire = 0x10000,
__memory_order_hle_release = 0x20000
};
constexpr memory_order
operator|(memory_order __m, __memory_order_modifier __mod)
{
return memory_order(int(__m) | int(__mod));
}
constexpr memory_order
operator&(memory_order __m, __memory_order_modifier __mod)
{
return memory_order(int(__m) & int(__mod));
}
constexpr memory_order
__cmpexch_failure_order2(memory_order __m) noexcept
{
return __m == memory_order_acq_rel ? memory_order_acquire
: __m == memory_order_release ? memory_order_relaxed : __m;
}
constexpr memory_order
__cmpexch_failure_order(memory_order __m) noexcept
{
return memory_order(__cmpexch_failure_order2(__m & __memory_order_mask)
| __memory_order_modifier(__m & __memory_order_modifier_mask));
}
constexpr bool
__is_valid_cmpexch_failure_order(memory_order __m) noexcept
{
return (__m & __memory_order_mask) != memory_order_release
&& (__m & __memory_order_mask) != memory_order_acq_rel;
}
template<typename _IntTp>
struct __atomic_base;
inline __attribute__((__always_inline__)) void
atomic_thread_fence(memory_order __m) noexcept
{ __atomic_thread_fence(int(__m)); }
inline __attribute__((__always_inline__)) void
atomic_signal_fence(memory_order __m) noexcept
{ __atomic_signal_fence(int(__m)); }
template<typename _Tp>
inline _Tp
kill_dependency(_Tp __y) noexcept
{
_Tp __ret(__y);
return __ret;
}
template<typename _Tp>
struct atomic;
template<typename _Tp>
struct atomic<_Tp*>;
typedef bool __atomic_flag_data_type;
extern "C" {
struct __atomic_flag_base
{
__atomic_flag_data_type _M_i ;
};
}
struct atomic_flag : public __atomic_flag_base
{
atomic_flag() noexcept = default;
~atomic_flag() noexcept = default;
atomic_flag(const atomic_flag&) = delete;
atomic_flag& operator=(const atomic_flag&) = delete;
atomic_flag& operator=(const atomic_flag&) volatile = delete;
constexpr atomic_flag(bool __i) noexcept
: __atomic_flag_base{ _S_init(__i) }
{ }
inline __attribute__((__always_inline__)) bool
test_and_set(memory_order __m = memory_order_seq_cst) noexcept
{
return __atomic_test_and_set (&_M_i, int(__m));
}
inline __attribute__((__always_inline__)) bool
test_and_set(memory_order __m = memory_order_seq_cst) volatile noexcept
{
return __atomic_test_and_set (&_M_i, int(__m));
}
inline __attribute__((__always_inline__)) void
clear(memory_order __m = memory_order_seq_cst) noexcept
{
memory_order __b __attribute__ ((__unused__))
= __m & __memory_order_mask;
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_consume)) __builtin_unreachable(); } while (false);
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_acquire)) __builtin_unreachable(); } while (false);
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_acq_rel)) __builtin_unreachable(); } while (false);
__atomic_clear (&_M_i, int(__m));
}
inline __attribute__((__always_inline__)) void
clear(memory_order __m = memory_order_seq_cst) volatile noexcept
{
memory_order __b __attribute__ ((__unused__))
= __m & __memory_order_mask;
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_consume)) __builtin_unreachable(); } while (false);
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_acquire)) __builtin_unreachable(); } while (false);
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_acq_rel)) __builtin_unreachable(); } while (false);
__atomic_clear (&_M_i, int(__m));
}
private:
static constexpr __atomic_flag_data_type
_S_init(bool __i)
{ return __i ? 1 : 0; }
};
template<typename _ITp>
struct __atomic_base
{
using value_type = _ITp;
using difference_type = value_type;
private:
typedef _ITp __int_type;
static constexpr int _S_alignment =
sizeof(_ITp) > alignof(_ITp) ? sizeof(_ITp) : alignof(_ITp);
alignas(_S_alignment) __int_type _M_i ;
public:
__atomic_base() noexcept = default;
~__atomic_base() noexcept = default;
__atomic_base(const __atomic_base&) = delete;
__atomic_base& operator=(const __atomic_base&) = delete;
__atomic_base& operator=(const __atomic_base&) volatile = delete;
constexpr __atomic_base(__int_type __i) noexcept : _M_i (__i) { }
operator __int_type() const noexcept
{ return load(); }
operator __int_type() const volatile noexcept
{ return load(); }
__int_type
operator=(__int_type __i) noexcept
{
store(__i);
return __i;
}
__int_type
operator=(__int_type __i) volatile noexcept
{
store(__i);
return __i;
}
__int_type
operator++(int) noexcept
{ return fetch_add(1); }
__int_type
operator++(int) volatile noexcept
{ return fetch_add(1); }
__int_type
operator--(int) noexcept
{ return fetch_sub(1); }
__int_type
operator--(int) volatile noexcept
{ return fetch_sub(1); }
__int_type
operator++() noexcept
{ return __atomic_add_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
__int_type
operator++() volatile noexcept
{ return __atomic_add_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
__int_type
operator--() noexcept
{ return __atomic_sub_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
__int_type
operator--() volatile noexcept
{ return __atomic_sub_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
__int_type
operator+=(__int_type __i) noexcept
{ return __atomic_add_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
__int_type
operator+=(__int_type __i) volatile noexcept
{ return __atomic_add_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
__int_type
operator-=(__int_type __i) noexcept
{ return __atomic_sub_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
__int_type
operator-=(__int_type __i) volatile noexcept
{ return __atomic_sub_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
__int_type
operator&=(__int_type __i) noexcept
{ return __atomic_and_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
__int_type
operator&=(__int_type __i) volatile noexcept
{ return __atomic_and_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
__int_type
operator|=(__int_type __i) noexcept
{ return __atomic_or_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
__int_type
operator|=(__int_type __i) volatile noexcept
{ return __atomic_or_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
__int_type
operator^=(__int_type __i) noexcept
{ return __atomic_xor_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
__int_type
operator^=(__int_type __i) volatile noexcept
{ return __atomic_xor_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
bool
is_lock_free() const noexcept
{
return __atomic_is_lock_free(sizeof(_M_i),
reinterpret_cast<void *>(-_S_alignment));
}
bool
is_lock_free() const volatile noexcept
{
return __atomic_is_lock_free(sizeof(_M_i),
reinterpret_cast<void *>(-_S_alignment));
}
inline __attribute__((__always_inline__)) void
store(__int_type __i, memory_order __m = memory_order_seq_cst) noexcept
{
memory_order __b __attribute__ ((__unused__))
= __m & __memory_order_mask;
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_acquire)) __builtin_unreachable(); } while (false);
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_acq_rel)) __builtin_unreachable(); } while (false);
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_consume)) __builtin_unreachable(); } while (false);
__atomic_store_n(&_M_i, __i, int(__m));
}
inline __attribute__((__always_inline__)) void
store(__int_type __i,
memory_order __m = memory_order_seq_cst) volatile noexcept
{
memory_order __b __attribute__ ((__unused__))
= __m & __memory_order_mask;
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_acquire)) __builtin_unreachable(); } while (false);
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_acq_rel)) __builtin_unreachable(); } while (false);
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_consume)) __builtin_unreachable(); } while (false);
__atomic_store_n(&_M_i, __i, int(__m));
}
inline __attribute__((__always_inline__)) __int_type
load(memory_order __m = memory_order_seq_cst) const noexcept
{
memory_order __b __attribute__ ((__unused__))
= __m & __memory_order_mask;
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_release)) __builtin_unreachable(); } while (false);
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_acq_rel)) __builtin_unreachable(); } while (false);
return __atomic_load_n(&_M_i, int(__m));
}
inline __attribute__((__always_inline__)) __int_type
load(memory_order __m = memory_order_seq_cst) const volatile noexcept
{
memory_order __b __attribute__ ((__unused__))
= __m & __memory_order_mask;
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_release)) __builtin_unreachable(); } while (false);
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_acq_rel)) __builtin_unreachable(); } while (false);
return __atomic_load_n(&_M_i, int(__m));
}
inline __attribute__((__always_inline__)) __int_type
exchange(__int_type __i,
memory_order __m = memory_order_seq_cst) noexcept
{
return __atomic_exchange_n(&_M_i, __i, int(__m));
}
inline __attribute__((__always_inline__)) __int_type
exchange(__int_type __i,
memory_order __m = memory_order_seq_cst) volatile noexcept
{
return __atomic_exchange_n(&_M_i, __i, int(__m));
}
inline __attribute__((__always_inline__)) bool
compare_exchange_weak(__int_type& __i1, __int_type __i2,
memory_order __m1, memory_order __m2) noexcept
{
do { if (std::__is_constant_evaluated() && !bool(__is_valid_cmpexch_failure_order(__m2))) __builtin_unreachable(); } while (false);
return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1,
int(__m1), int(__m2));
}
inline __attribute__((__always_inline__)) bool
compare_exchange_weak(__int_type& __i1, __int_type __i2,
memory_order __m1,
memory_order __m2) volatile noexcept
{
do { if (std::__is_constant_evaluated() && !bool(__is_valid_cmpexch_failure_order(__m2))) __builtin_unreachable(); } while (false);
return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1,
int(__m1), int(__m2));
}
inline __attribute__((__always_inline__)) bool
compare_exchange_weak(__int_type& __i1, __int_type __i2,
memory_order __m = memory_order_seq_cst) noexcept
{
return compare_exchange_weak(__i1, __i2, __m,
__cmpexch_failure_order(__m));
}
inline __attribute__((__always_inline__)) bool
compare_exchange_weak(__int_type& __i1, __int_type __i2,
memory_order __m = memory_order_seq_cst) volatile noexcept
{
return compare_exchange_weak(__i1, __i2, __m,
__cmpexch_failure_order(__m));
}
inline __attribute__((__always_inline__)) bool
compare_exchange_strong(__int_type& __i1, __int_type __i2,
memory_order __m1, memory_order __m2) noexcept
{
do { if (std::__is_constant_evaluated() && !bool(__is_valid_cmpexch_failure_order(__m2))) __builtin_unreachable(); } while (false);
return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0,
int(__m1), int(__m2));
}
inline __attribute__((__always_inline__)) bool
compare_exchange_strong(__int_type& __i1, __int_type __i2,
memory_order __m1,
memory_order __m2) volatile noexcept
{
do { if (std::__is_constant_evaluated() && !bool(__is_valid_cmpexch_failure_order(__m2))) __builtin_unreachable(); } while (false);
return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0,
int(__m1), int(__m2));
}
inline __attribute__((__always_inline__)) bool
compare_exchange_strong(__int_type& __i1, __int_type __i2,
memory_order __m = memory_order_seq_cst) noexcept
{
return compare_exchange_strong(__i1, __i2, __m,
__cmpexch_failure_order(__m));
}
inline __attribute__((__always_inline__)) bool
compare_exchange_strong(__int_type& __i1, __int_type __i2,
memory_order __m = memory_order_seq_cst) volatile noexcept
{
return compare_exchange_strong(__i1, __i2, __m,
__cmpexch_failure_order(__m));
}
inline __attribute__((__always_inline__)) __int_type
fetch_add(__int_type __i,
memory_order __m = memory_order_seq_cst) noexcept
{ return __atomic_fetch_add(&_M_i, __i, int(__m)); }
inline __attribute__((__always_inline__)) __int_type
fetch_add(__int_type __i,
memory_order __m = memory_order_seq_cst) volatile noexcept
{ return __atomic_fetch_add(&_M_i, __i, int(__m)); }
inline __attribute__((__always_inline__)) __int_type
fetch_sub(__int_type __i,
memory_order __m = memory_order_seq_cst) noexcept
{ return __atomic_fetch_sub(&_M_i, __i, int(__m)); }
inline __attribute__((__always_inline__)) __int_type
fetch_sub(__int_type __i,
memory_order __m = memory_order_seq_cst) volatile noexcept
{ return __atomic_fetch_sub(&_M_i, __i, int(__m)); }
inline __attribute__((__always_inline__)) __int_type
fetch_and(__int_type __i,
memory_order __m = memory_order_seq_cst) noexcept
{ return __atomic_fetch_and(&_M_i, __i, int(__m)); }
inline __attribute__((__always_inline__)) __int_type
fetch_and(__int_type __i,
memory_order __m = memory_order_seq_cst) volatile noexcept
{ return __atomic_fetch_and(&_M_i, __i, int(__m)); }
inline __attribute__((__always_inline__)) __int_type
fetch_or(__int_type __i,
memory_order __m = memory_order_seq_cst) noexcept
{ return __atomic_fetch_or(&_M_i, __i, int(__m)); }
inline __attribute__((__always_inline__)) __int_type
fetch_or(__int_type __i,
memory_order __m = memory_order_seq_cst) volatile noexcept
{ return __atomic_fetch_or(&_M_i, __i, int(__m)); }
inline __attribute__((__always_inline__)) __int_type
fetch_xor(__int_type __i,
memory_order __m = memory_order_seq_cst) noexcept
{ return __atomic_fetch_xor(&_M_i, __i, int(__m)); }
inline __attribute__((__always_inline__)) __int_type
fetch_xor(__int_type __i,
memory_order __m = memory_order_seq_cst) volatile noexcept
{ return __atomic_fetch_xor(&_M_i, __i, int(__m)); }
};
template<typename _PTp>
struct __atomic_base<_PTp*>
{
private:
typedef _PTp* __pointer_type;
__pointer_type _M_p ;
constexpr ptrdiff_t
_M_type_size(ptrdiff_t __d) const { return __d * sizeof(_PTp); }
constexpr ptrdiff_t
_M_type_size(ptrdiff_t __d) const volatile { return __d * sizeof(_PTp); }
public:
__atomic_base() noexcept = default;
~__atomic_base() noexcept = default;
__atomic_base(const __atomic_base&) = delete;
__atomic_base& operator=(const __atomic_base&) = delete;
__atomic_base& operator=(const __atomic_base&) volatile = delete;
constexpr __atomic_base(__pointer_type __p) noexcept : _M_p (__p) { }
operator __pointer_type() const noexcept
{ return load(); }
operator __pointer_type() const volatile noexcept
{ return load(); }
__pointer_type
operator=(__pointer_type __p) noexcept
{
store(__p);
return __p;
}
__pointer_type
operator=(__pointer_type __p) volatile noexcept
{
store(__p);
return __p;
}
__pointer_type
operator++(int) noexcept
{ return fetch_add(1); }
__pointer_type
operator++(int) volatile noexcept
{ return fetch_add(1); }
__pointer_type
operator--(int) noexcept
{ return fetch_sub(1); }
__pointer_type
operator--(int) volatile noexcept
{ return fetch_sub(1); }
__pointer_type
operator++() noexcept
{ return __atomic_add_fetch(&_M_p, _M_type_size(1),
int(memory_order_seq_cst)); }
__pointer_type
operator++() volatile noexcept
{ return __atomic_add_fetch(&_M_p, _M_type_size(1),
int(memory_order_seq_cst)); }
__pointer_type
operator--() noexcept
{ return __atomic_sub_fetch(&_M_p, _M_type_size(1),
int(memory_order_seq_cst)); }
__pointer_type
operator--() volatile noexcept
{ return __atomic_sub_fetch(&_M_p, _M_type_size(1),
int(memory_order_seq_cst)); }
__pointer_type
operator+=(ptrdiff_t __d) noexcept
{ return __atomic_add_fetch(&_M_p, _M_type_size(__d),
int(memory_order_seq_cst)); }
__pointer_type
operator+=(ptrdiff_t __d) volatile noexcept
{ return __atomic_add_fetch(&_M_p, _M_type_size(__d),
int(memory_order_seq_cst)); }
__pointer_type
operator-=(ptrdiff_t __d) noexcept
{ return __atomic_sub_fetch(&_M_p, _M_type_size(__d),
int(memory_order_seq_cst)); }
__pointer_type
operator-=(ptrdiff_t __d) volatile noexcept
{ return __atomic_sub_fetch(&_M_p, _M_type_size(__d),
int(memory_order_seq_cst)); }
bool
is_lock_free() const noexcept
{
return __atomic_is_lock_free(sizeof(_M_p),
reinterpret_cast<void *>(-__alignof(_M_p)));
}
bool
is_lock_free() const volatile noexcept
{
return __atomic_is_lock_free(sizeof(_M_p),
reinterpret_cast<void *>(-__alignof(_M_p)));
}
inline __attribute__((__always_inline__)) void
store(__pointer_type __p,
memory_order __m = memory_order_seq_cst) noexcept
{
memory_order __b __attribute__ ((__unused__))
= __m & __memory_order_mask;
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_acquire)) __builtin_unreachable(); } while (false);
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_acq_rel)) __builtin_unreachable(); } while (false);
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_consume)) __builtin_unreachable(); } while (false);
__atomic_store_n(&_M_p, __p, int(__m));
}
inline __attribute__((__always_inline__)) void
store(__pointer_type __p,
memory_order __m = memory_order_seq_cst) volatile noexcept
{
memory_order __b __attribute__ ((__unused__))
= __m & __memory_order_mask;
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_acquire)) __builtin_unreachable(); } while (false);
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_acq_rel)) __builtin_unreachable(); } while (false);
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_consume)) __builtin_unreachable(); } while (false);
__atomic_store_n(&_M_p, __p, int(__m));
}
inline __attribute__((__always_inline__)) __pointer_type
load(memory_order __m = memory_order_seq_cst) const noexcept
{
memory_order __b __attribute__ ((__unused__))
= __m & __memory_order_mask;
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_release)) __builtin_unreachable(); } while (false);
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_acq_rel)) __builtin_unreachable(); } while (false);
return __atomic_load_n(&_M_p, int(__m));
}
inline __attribute__((__always_inline__)) __pointer_type
load(memory_order __m = memory_order_seq_cst) const volatile noexcept
{
memory_order __b __attribute__ ((__unused__))
= __m & __memory_order_mask;
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_release)) __builtin_unreachable(); } while (false);
do { if (std::__is_constant_evaluated() && !bool(__b != memory_order_acq_rel)) __builtin_unreachable(); } while (false);
return __atomic_load_n(&_M_p, int(__m));
}
inline __attribute__((__always_inline__)) __pointer_type
exchange(__pointer_type __p,
memory_order __m = memory_order_seq_cst) noexcept
{
return __atomic_exchange_n(&_M_p, __p, int(__m));
}
inline __attribute__((__always_inline__)) __pointer_type
exchange(__pointer_type __p,
memory_order __m = memory_order_seq_cst) volatile noexcept
{
return __atomic_exchange_n(&_M_p, __p, int(__m));
}
inline __attribute__((__always_inline__)) bool
compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
memory_order __m1,
memory_order __m2) noexcept
{
do { if (std::__is_constant_evaluated() && !bool(__is_valid_cmpexch_failure_order(__m2))) __builtin_unreachable(); } while (false);
return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 1,
int(__m1), int(__m2));
}
inline __attribute__((__always_inline__)) bool
compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
memory_order __m1,
memory_order __m2) volatile noexcept
{
do { if (std::__is_constant_evaluated() && !bool(__is_valid_cmpexch_failure_order(__m2))) __builtin_unreachable(); } while (false);
return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 1,
int(__m1), int(__m2));
}
inline __attribute__((__always_inline__)) bool
compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
memory_order __m1,
memory_order __m2) noexcept
{
do { if (std::__is_constant_evaluated() && !bool(__is_valid_cmpexch_failure_order(__m2))) __builtin_unreachable(); } while (false);
return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0,
int(__m1), int(__m2));
}
inline __attribute__((__always_inline__)) bool
compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
memory_order __m1,
memory_order __m2) volatile noexcept
{
do { if (std::__is_constant_evaluated() && !bool(__is_valid_cmpexch_failure_order(__m2))) __builtin_unreachable(); } while (false);
return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0,
int(__m1), int(__m2));
}
inline __attribute__((__always_inline__)) __pointer_type
fetch_add(ptrdiff_t __d,
memory_order __m = memory_order_seq_cst) noexcept
{ return __atomic_fetch_add(&_M_p, _M_type_size(__d), int(__m)); }
inline __attribute__((__always_inline__)) __pointer_type
fetch_add(ptrdiff_t __d,
memory_order __m = memory_order_seq_cst) volatile noexcept
{ return __atomic_fetch_add(&_M_p, _M_type_size(__d), int(__m)); }
inline __attribute__((__always_inline__)) __pointer_type
fetch_sub(ptrdiff_t __d,
memory_order __m = memory_order_seq_cst) noexcept
{ return __atomic_fetch_sub(&_M_p, _M_type_size(__d), int(__m)); }
inline __attribute__((__always_inline__)) __pointer_type
fetch_sub(ptrdiff_t __d,
memory_order __m = memory_order_seq_cst) volatile noexcept
{ return __atomic_fetch_sub(&_M_p, _M_type_size(__d), int(__m)); }
};
namespace __atomic_impl
{
template<typename _Tp>
constexpr bool
__maybe_has_padding()
{
return !__has_unique_object_representations(_Tp)
&& !is_same<_Tp, float>::value && !is_same<_Tp, double>::value;
}
template<typename _Tp>
inline __attribute__((__always_inline__)) _Tp*
__clear_padding(_Tp& __val) noexcept
{
auto* __ptr = std::__addressof(__val);
if constexpr (__atomic_impl::__maybe_has_padding<_Tp>())
__builtin_clear_padding(__ptr);
return __ptr;
}
template<typename _Tp>
using _Val = typename remove_volatile<_Tp>::type;
template<typename _Tp>
inline __attribute__((__always_inline__)) bool
__compare_exchange(_Tp& __val, _Val<_Tp>& __e, _Val<_Tp>& __i,
bool __is_weak,
memory_order __s, memory_order __f) noexcept
{
do { if (std::__is_constant_evaluated() && !bool(__is_valid_cmpexch_failure_order(__f))) __builtin_unreachable(); } while (false);
using _Vp = _Val<_Tp>;
if constexpr (__atomic_impl::__maybe_has_padding<_Vp>())
{
alignas(_Vp) unsigned char __buf[sizeof(_Vp)];
_Vp* __exp = ::new((void*)__buf) _Vp(__e);
__atomic_impl::__clear_padding(*__exp);
if (__atomic_compare_exchange(std::__addressof(__val), __exp,
__atomic_impl::__clear_padding(__i),
__is_weak, int(__s), int(__f)))
return true;
__builtin_memcpy(std::__addressof(__e), __exp, sizeof(_Vp));
return false;
}
else
return __atomic_compare_exchange(std::__addressof(__val),
std::__addressof(__e),
std::__addressof(__i),
__is_weak, int(__s), int(__f));
}
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Tp>
struct atomic;
template<>
struct atomic<bool>
{
using value_type = bool;
private:
__atomic_base<bool> _M_base;
public:
atomic() noexcept = default;
~atomic() noexcept = default;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
constexpr atomic(bool __i) noexcept : _M_base(__i) { }
bool
operator=(bool __i) noexcept
{ return _M_base.operator=(__i); }
bool
operator=(bool __i) volatile noexcept
{ return _M_base.operator=(__i); }
operator bool() const noexcept
{ return _M_base.load(); }
operator bool() const volatile noexcept
{ return _M_base.load(); }
bool
is_lock_free() const noexcept { return _M_base.is_lock_free(); }
bool
is_lock_free() const volatile noexcept { return _M_base.is_lock_free(); }
static constexpr bool is_always_lock_free = 2 == 2;
void
store(bool __i, memory_order __m = memory_order_seq_cst) noexcept
{ _M_base.store(__i, __m); }
void
store(bool __i, memory_order __m = memory_order_seq_cst) volatile noexcept
{ _M_base.store(__i, __m); }
bool
load(memory_order __m = memory_order_seq_cst) const noexcept
{ return _M_base.load(__m); }
bool
load(memory_order __m = memory_order_seq_cst) const volatile noexcept
{ return _M_base.load(__m); }
bool
exchange(bool __i, memory_order __m = memory_order_seq_cst) noexcept
{ return _M_base.exchange(__i, __m); }
bool
exchange(bool __i,
memory_order __m = memory_order_seq_cst) volatile noexcept
{ return _M_base.exchange(__i, __m); }
bool
compare_exchange_weak(bool& __i1, bool __i2, memory_order __m1,
memory_order __m2) noexcept
{ return _M_base.compare_exchange_weak(__i1, __i2, __m1, __m2); }
bool
compare_exchange_weak(bool& __i1, bool __i2, memory_order __m1,
memory_order __m2) volatile noexcept
{ return _M_base.compare_exchange_weak(__i1, __i2, __m1, __m2); }
bool
compare_exchange_weak(bool& __i1, bool __i2,
memory_order __m = memory_order_seq_cst) noexcept
{ return _M_base.compare_exchange_weak(__i1, __i2, __m); }
bool
compare_exchange_weak(bool& __i1, bool __i2,
memory_order __m = memory_order_seq_cst) volatile noexcept
{ return _M_base.compare_exchange_weak(__i1, __i2, __m); }
bool
compare_exchange_strong(bool& __i1, bool __i2, memory_order __m1,
memory_order __m2) noexcept
{ return _M_base.compare_exchange_strong(__i1, __i2, __m1, __m2); }
bool
compare_exchange_strong(bool& __i1, bool __i2, memory_order __m1,
memory_order __m2) volatile noexcept
{ return _M_base.compare_exchange_strong(__i1, __i2, __m1, __m2); }
bool
compare_exchange_strong(bool& __i1, bool __i2,
memory_order __m = memory_order_seq_cst) noexcept
{ return _M_base.compare_exchange_strong(__i1, __i2, __m); }
bool
compare_exchange_strong(bool& __i1, bool __i2,
memory_order __m = memory_order_seq_cst) volatile noexcept
{ return _M_base.compare_exchange_strong(__i1, __i2, __m); }
};
template<typename _Tp>
struct atomic
{
using value_type = _Tp;
private:
static constexpr int _S_min_alignment
= (sizeof(_Tp) & (sizeof(_Tp) - 1)) || sizeof(_Tp) > 16
? 0 : sizeof(_Tp);
static constexpr int _S_alignment
= _S_min_alignment > alignof(_Tp) ? _S_min_alignment : alignof(_Tp);
alignas(_S_alignment) _Tp _M_i ;
static_assert(__is_trivially_copyable(_Tp),
"std::atomic requires a trivially copyable type");
static_assert(sizeof(_Tp) > 0,
"Incomplete or zero-sized types are not supported");
public:
atomic() = default;
~atomic() noexcept = default;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
constexpr atomic(_Tp __i) noexcept : _M_i(__i)
{
if constexpr (__atomic_impl::__maybe_has_padding<_Tp>())
__builtin_clear_padding(std::__addressof(_M_i));
}
operator _Tp() const noexcept
{ return load(); }
operator _Tp() const volatile noexcept
{ return load(); }
_Tp
operator=(_Tp __i) noexcept
{ store(__i); return __i; }
_Tp
operator=(_Tp __i) volatile noexcept
{ store(__i); return __i; }
bool
is_lock_free() const noexcept
{
return __atomic_is_lock_free(sizeof(_M_i),
reinterpret_cast<void *>(-_S_alignment));
}
bool
is_lock_free() const volatile noexcept
{
return __atomic_is_lock_free(sizeof(_M_i),
reinterpret_cast<void *>(-_S_alignment));
}
static constexpr bool is_always_lock_free
= __atomic_always_lock_free(sizeof(_M_i), 0);
void
store(_Tp __i, memory_order __m = memory_order_seq_cst) noexcept
{
__atomic_store(std::__addressof(_M_i),
__atomic_impl::__clear_padding(__i),
int(__m));
}
void
store(_Tp __i, memory_order __m = memory_order_seq_cst) volatile noexcept
{
__atomic_store(std::__addressof(_M_i),
__atomic_impl::__clear_padding(__i),
int(__m));
}
_Tp
load(memory_order __m = memory_order_seq_cst) const noexcept
{
alignas(_Tp) unsigned char __buf[sizeof(_Tp)];
_Tp* __ptr = reinterpret_cast<_Tp*>(__buf);
__atomic_load(std::__addressof(_M_i), __ptr, int(__m));
return *__ptr;
}
_Tp
load(memory_order __m = memory_order_seq_cst) const volatile noexcept
{
alignas(_Tp) unsigned char __buf[sizeof(_Tp)];
_Tp* __ptr = reinterpret_cast<_Tp*>(__buf);
__atomic_load(std::__addressof(_M_i), __ptr, int(__m));
return *__ptr;
}
_Tp
exchange(_Tp __i, memory_order __m = memory_order_seq_cst) noexcept
{
alignas(_Tp) unsigned char __buf[sizeof(_Tp)];
_Tp* __ptr = reinterpret_cast<_Tp*>(__buf);
__atomic_exchange(std::__addressof(_M_i),
__atomic_impl::__clear_padding(__i),
__ptr, int(__m));
return *__ptr;
}
_Tp
exchange(_Tp __i,
memory_order __m = memory_order_seq_cst) volatile noexcept
{
alignas(_Tp) unsigned char __buf[sizeof(_Tp)];
_Tp* __ptr = reinterpret_cast<_Tp*>(__buf);
__atomic_exchange(std::__addressof(_M_i),
__atomic_impl::__clear_padding(__i),
__ptr, int(__m));
return *__ptr;
}
bool
compare_exchange_weak(_Tp& __e, _Tp __i, memory_order __s,
memory_order __f) noexcept
{
return __atomic_impl::__compare_exchange(_M_i, __e, __i, true,
__s, __f);
}
bool
compare_exchange_weak(_Tp& __e, _Tp __i, memory_order __s,
memory_order __f) volatile noexcept
{
return __atomic_impl::__compare_exchange(_M_i, __e, __i, true,
__s, __f);
}
bool
compare_exchange_weak(_Tp& __e, _Tp __i,
memory_order __m = memory_order_seq_cst) noexcept
{ return compare_exchange_weak(__e, __i, __m,
__cmpexch_failure_order(__m)); }
bool
compare_exchange_weak(_Tp& __e, _Tp __i,
memory_order __m = memory_order_seq_cst) volatile noexcept
{ return compare_exchange_weak(__e, __i, __m,
__cmpexch_failure_order(__m)); }
bool
compare_exchange_strong(_Tp& __e, _Tp __i, memory_order __s,
memory_order __f) noexcept
{
return __atomic_impl::__compare_exchange(_M_i, __e, __i, false,
__s, __f);
}
bool
compare_exchange_strong(_Tp& __e, _Tp __i, memory_order __s,
memory_order __f) volatile noexcept
{
return __atomic_impl::__compare_exchange(_M_i, __e, __i, false,
__s, __f);
}
bool
compare_exchange_strong(_Tp& __e, _Tp __i,
memory_order __m = memory_order_seq_cst) noexcept
{ return compare_exchange_strong(__e, __i, __m,
__cmpexch_failure_order(__m)); }
bool
compare_exchange_strong(_Tp& __e, _Tp __i,
memory_order __m = memory_order_seq_cst) volatile noexcept
{ return compare_exchange_strong(__e, __i, __m,
__cmpexch_failure_order(__m)); }
};
template<typename _Tp>
struct atomic<_Tp*>
{
using value_type = _Tp*;
using difference_type = ptrdiff_t;
typedef _Tp* __pointer_type;
typedef __atomic_base<_Tp*> __base_type;
__base_type _M_b;
atomic() noexcept = default;
~atomic() noexcept = default;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
constexpr atomic(__pointer_type __p) noexcept : _M_b(__p) { }
operator __pointer_type() const noexcept
{ return __pointer_type(_M_b); }
operator __pointer_type() const volatile noexcept
{ return __pointer_type(_M_b); }
__pointer_type
operator=(__pointer_type __p) noexcept
{ return _M_b.operator=(__p); }
__pointer_type
operator=(__pointer_type __p) volatile noexcept
{ return _M_b.operator=(__p); }
__pointer_type
operator++(int) noexcept
{
static_assert( is_object<_Tp>::value, "pointer to object type" );
return _M_b++;
}
__pointer_type
operator++(int) volatile noexcept
{
static_assert( is_object<_Tp>::value, "pointer to object type" );
return _M_b++;
}
__pointer_type
operator--(int) noexcept
{
static_assert( is_object<_Tp>::value, "pointer to object type" );
return _M_b--;
}
__pointer_type
operator--(int) volatile noexcept
{
static_assert( is_object<_Tp>::value, "pointer to object type" );
return _M_b--;
}
__pointer_type
operator++() noexcept
{
static_assert( is_object<_Tp>::value, "pointer to object type" );
return ++_M_b;
}
__pointer_type
operator++() volatile noexcept
{
static_assert( is_object<_Tp>::value, "pointer to object type" );
return ++_M_b;
}
__pointer_type
operator--() noexcept
{
static_assert( is_object<_Tp>::value, "pointer to object type" );
return --_M_b;
}
__pointer_type
operator--() volatile noexcept
{
static_assert( is_object<_Tp>::value, "pointer to object type" );
return --_M_b;
}
__pointer_type
operator+=(ptrdiff_t __d) noexcept
{
static_assert( is_object<_Tp>::value, "pointer to object type" );
return _M_b.operator+=(__d);
}
__pointer_type
operator+=(ptrdiff_t __d) volatile noexcept
{
static_assert( is_object<_Tp>::value, "pointer to object type" );
return _M_b.operator+=(__d);
}
__pointer_type
operator-=(ptrdiff_t __d) noexcept
{
static_assert( is_object<_Tp>::value, "pointer to object type" );
return _M_b.operator-=(__d);
}
__pointer_type
operator-=(ptrdiff_t __d) volatile noexcept
{
static_assert( is_object<_Tp>::value, "pointer to object type" );
return _M_b.operator-=(__d);
}
bool
is_lock_free() const noexcept
{ return _M_b.is_lock_free(); }
bool
is_lock_free() const volatile noexcept
{ return _M_b.is_lock_free(); }
static constexpr bool is_always_lock_free
= 2 == 2;
void
store(__pointer_type __p,
memory_order __m = memory_order_seq_cst) noexcept
{ return _M_b.store(__p, __m); }
void
store(__pointer_type __p,
memory_order __m = memory_order_seq_cst) volatile noexcept
{ return _M_b.store(__p, __m); }
__pointer_type
load(memory_order __m = memory_order_seq_cst) const noexcept
{ return _M_b.load(__m); }
__pointer_type
load(memory_order __m = memory_order_seq_cst) const volatile noexcept
{ return _M_b.load(__m); }
__pointer_type
exchange(__pointer_type __p,
memory_order __m = memory_order_seq_cst) noexcept
{ return _M_b.exchange(__p, __m); }
__pointer_type
exchange(__pointer_type __p,
memory_order __m = memory_order_seq_cst) volatile noexcept
{ return _M_b.exchange(__p, __m); }
bool
compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
memory_order __m1, memory_order __m2) noexcept
{ return _M_b.compare_exchange_weak(__p1, __p2, __m1, __m2); }
bool
compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
memory_order __m1,
memory_order __m2) volatile noexcept
{ return _M_b.compare_exchange_weak(__p1, __p2, __m1, __m2); }
bool
compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
memory_order __m = memory_order_seq_cst) noexcept
{
return compare_exchange_weak(__p1, __p2, __m,
__cmpexch_failure_order(__m));
}
bool
compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
memory_order __m = memory_order_seq_cst) volatile noexcept
{
return compare_exchange_weak(__p1, __p2, __m,
__cmpexch_failure_order(__m));
}
bool
compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
memory_order __m1, memory_order __m2) noexcept
{ return _M_b.compare_exchange_strong(__p1, __p2, __m1, __m2); }
bool
compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
memory_order __m1,
memory_order __m2) volatile noexcept
{ return _M_b.compare_exchange_strong(__p1, __p2, __m1, __m2); }
bool
compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
memory_order __m = memory_order_seq_cst) noexcept
{
return _M_b.compare_exchange_strong(__p1, __p2, __m,
__cmpexch_failure_order(__m));
}
bool
compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
memory_order __m = memory_order_seq_cst) volatile noexcept
{
return _M_b.compare_exchange_strong(__p1, __p2, __m,
__cmpexch_failure_order(__m));
}
__pointer_type
fetch_add(ptrdiff_t __d,
memory_order __m = memory_order_seq_cst) noexcept
{
static_assert( is_object<_Tp>::value, "pointer to object type" );
return _M_b.fetch_add(__d, __m);
}
__pointer_type
fetch_add(ptrdiff_t __d,
memory_order __m = memory_order_seq_cst) volatile noexcept
{
static_assert( is_object<_Tp>::value, "pointer to object type" );
return _M_b.fetch_add(__d, __m);
}
__pointer_type
fetch_sub(ptrdiff_t __d,
memory_order __m = memory_order_seq_cst) noexcept
{
static_assert( is_object<_Tp>::value, "pointer to object type" );
return _M_b.fetch_sub(__d, __m);
}
__pointer_type
fetch_sub(ptrdiff_t __d,
memory_order __m = memory_order_seq_cst) volatile noexcept
{
static_assert( is_object<_Tp>::value, "pointer to object type" );
return _M_b.fetch_sub(__d, __m);
}
};
template<>
struct atomic<char> : __atomic_base<char>
{
typedef char __integral_type;
typedef __atomic_base<char> __base_type;
atomic() noexcept = default;
~atomic() noexcept = default;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
using __base_type::operator __integral_type;
using __base_type::operator=;
static constexpr bool is_always_lock_free = 2 == 2;
};
template<>
struct atomic<signed char> : __atomic_base<signed char>
{
typedef signed char __integral_type;
typedef __atomic_base<signed char> __base_type;
atomic() noexcept= default;
~atomic() noexcept = default;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
using __base_type::operator __integral_type;
using __base_type::operator=;
static constexpr bool is_always_lock_free = 2 == 2;
};
template<>
struct atomic<unsigned char> : __atomic_base<unsigned char>
{
typedef unsigned char __integral_type;
typedef __atomic_base<unsigned char> __base_type;
atomic() noexcept= default;
~atomic() noexcept = default;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
using __base_type::operator __integral_type;
using __base_type::operator=;
static constexpr bool is_always_lock_free = 2 == 2;
};
template<>
struct atomic<short> : __atomic_base<short>
{
typedef short __integral_type;
typedef __atomic_base<short> __base_type;
atomic() noexcept = default;
~atomic() noexcept = default;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
using __base_type::operator __integral_type;
using __base_type::operator=;
static constexpr bool is_always_lock_free = 2 == 2;
};
template<>
struct atomic<unsigned short> : __atomic_base<unsigned short>
{
typedef unsigned short __integral_type;
typedef __atomic_base<unsigned short> __base_type;
atomic() noexcept = default;
~atomic() noexcept = default;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
using __base_type::operator __integral_type;
using __base_type::operator=;
static constexpr bool is_always_lock_free = 2 == 2;
};
template<>
struct atomic<int> : __atomic_base<int>
{
typedef int __integral_type;
typedef __atomic_base<int> __base_type;
atomic() noexcept = default;
~atomic() noexcept = default;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
using __base_type::operator __integral_type;
using __base_type::operator=;
static constexpr bool is_always_lock_free = 2 == 2;
};
template<>
struct atomic<unsigned int> : __atomic_base<unsigned int>
{
typedef unsigned int __integral_type;
typedef __atomic_base<unsigned int> __base_type;
atomic() noexcept = default;
~atomic() noexcept = default;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
using __base_type::operator __integral_type;
using __base_type::operator=;
static constexpr bool is_always_lock_free = 2 == 2;
};
template<>
struct atomic<long> : __atomic_base<long>
{
typedef long __integral_type;
typedef __atomic_base<long> __base_type;
atomic() noexcept = default;
~atomic() noexcept = default;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
using __base_type::operator __integral_type;
using __base_type::operator=;
static constexpr bool is_always_lock_free = 2 == 2;
};
template<>
struct atomic<unsigned long> : __atomic_base<unsigned long>
{
typedef unsigned long __integral_type;
typedef __atomic_base<unsigned long> __base_type;
atomic() noexcept = default;
~atomic() noexcept = default;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
using __base_type::operator __integral_type;
using __base_type::operator=;
static constexpr bool is_always_lock_free = 2 == 2;
};
template<>
struct atomic<long long> : __atomic_base<long long>
{
typedef long long __integral_type;
typedef __atomic_base<long long> __base_type;
atomic() noexcept = default;
~atomic() noexcept = default;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
using __base_type::operator __integral_type;
using __base_type::operator=;
static constexpr bool is_always_lock_free = 2 == 2;
};
template<>
struct atomic<unsigned long long> : __atomic_base<unsigned long long>
{
typedef unsigned long long __integral_type;
typedef __atomic_base<unsigned long long> __base_type;
atomic() noexcept = default;
~atomic() noexcept = default;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
using __base_type::operator __integral_type;
using __base_type::operator=;
static constexpr bool is_always_lock_free = 2 == 2;
};
template<>
struct atomic<wchar_t> : __atomic_base<wchar_t>
{
typedef wchar_t __integral_type;
typedef __atomic_base<wchar_t> __base_type;
atomic() noexcept = default;
~atomic() noexcept = default;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
using __base_type::operator __integral_type;
using __base_type::operator=;
static constexpr bool is_always_lock_free = 2 == 2;
};
template<>
struct atomic<char16_t> : __atomic_base<char16_t>
{
typedef char16_t __integral_type;
typedef __atomic_base<char16_t> __base_type;
atomic() noexcept = default;
~atomic() noexcept = default;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
using __base_type::operator __integral_type;
using __base_type::operator=;
static constexpr bool is_always_lock_free
= 2 == 2;
};
template<>
struct atomic<char32_t> : __atomic_base<char32_t>
{
typedef char32_t __integral_type;
typedef __atomic_base<char32_t> __base_type;
atomic() noexcept = default;
~atomic() noexcept = default;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
constexpr atomic(__integral_type __i) noexcept : __base_type(__i) { }
using __base_type::operator __integral_type;
using __base_type::operator=;
static constexpr bool is_always_lock_free
= 2 == 2;
};
typedef atomic<bool> atomic_bool;
typedef atomic<char> atomic_char;
typedef atomic<signed char> atomic_schar;
typedef atomic<unsigned char> atomic_uchar;
typedef atomic<short> atomic_short;
typedef atomic<unsigned short> atomic_ushort;
typedef atomic<int> atomic_int;
typedef atomic<unsigned int> atomic_uint;
typedef atomic<long> atomic_long;
typedef atomic<unsigned long> atomic_ulong;
typedef atomic<long long> atomic_llong;
typedef atomic<unsigned long long> atomic_ullong;
typedef atomic<wchar_t> atomic_wchar_t;
typedef atomic<char16_t> atomic_char16_t;
typedef atomic<char32_t> atomic_char32_t;
typedef atomic<int8_t> atomic_int8_t;
typedef atomic<uint8_t> atomic_uint8_t;
typedef atomic<int16_t> atomic_int16_t;
typedef atomic<uint16_t> atomic_uint16_t;
typedef atomic<int32_t> atomic_int32_t;
typedef atomic<uint32_t> atomic_uint32_t;
typedef atomic<int64_t> atomic_int64_t;
typedef atomic<uint64_t> atomic_uint64_t;
typedef atomic<int_least8_t> atomic_int_least8_t;
typedef atomic<uint_least8_t> atomic_uint_least8_t;
typedef atomic<int_least16_t> atomic_int_least16_t;
typedef atomic<uint_least16_t> atomic_uint_least16_t;
typedef atomic<int_least32_t> atomic_int_least32_t;
typedef atomic<uint_least32_t> atomic_uint_least32_t;
typedef atomic<int_least64_t> atomic_int_least64_t;
typedef atomic<uint_least64_t> atomic_uint_least64_t;
typedef atomic<int_fast8_t> atomic_int_fast8_t;
typedef atomic<uint_fast8_t> atomic_uint_fast8_t;
typedef atomic<int_fast16_t> atomic_int_fast16_t;
typedef atomic<uint_fast16_t> atomic_uint_fast16_t;
typedef atomic<int_fast32_t> atomic_int_fast32_t;
typedef atomic<uint_fast32_t> atomic_uint_fast32_t;
typedef atomic<int_fast64_t> atomic_int_fast64_t;
typedef atomic<uint_fast64_t> atomic_uint_fast64_t;
typedef atomic<intptr_t> atomic_intptr_t;
typedef atomic<uintptr_t> atomic_uintptr_t;
typedef atomic<size_t> atomic_size_t;
typedef atomic<ptrdiff_t> atomic_ptrdiff_t;
typedef atomic<intmax_t> atomic_intmax_t;
typedef atomic<uintmax_t> atomic_uintmax_t;
inline bool
atomic_flag_test_and_set_explicit(atomic_flag* __a,
memory_order __m) noexcept
{ return __a->test_and_set(__m); }
inline bool
atomic_flag_test_and_set_explicit(volatile atomic_flag* __a,
memory_order __m) noexcept
{ return __a->test_and_set(__m); }
inline void
atomic_flag_clear_explicit(atomic_flag* __a, memory_order __m) noexcept
{ __a->clear(__m); }
inline void
atomic_flag_clear_explicit(volatile atomic_flag* __a,
memory_order __m) noexcept
{ __a->clear(__m); }
inline bool
atomic_flag_test_and_set(atomic_flag* __a) noexcept
{ return atomic_flag_test_and_set_explicit(__a, memory_order_seq_cst); }
inline bool
atomic_flag_test_and_set(volatile atomic_flag* __a) noexcept
{ return atomic_flag_test_and_set_explicit(__a, memory_order_seq_cst); }
inline void
atomic_flag_clear(atomic_flag* __a) noexcept
{ atomic_flag_clear_explicit(__a, memory_order_seq_cst); }
inline void
atomic_flag_clear(volatile atomic_flag* __a) noexcept
{ atomic_flag_clear_explicit(__a, memory_order_seq_cst); }
template<typename _Tp>
using __atomic_val_t = __type_identity_t<_Tp>;
template<typename _Tp>
using __atomic_diff_t = typename atomic<_Tp>::difference_type;
template<typename _ITp>
inline bool
atomic_is_lock_free(const atomic<_ITp>* __a) noexcept
{ return __a->is_lock_free(); }
template<typename _ITp>
inline bool
atomic_is_lock_free(const volatile atomic<_ITp>* __a) noexcept
{ return __a->is_lock_free(); }
template<typename _ITp>
inline void
atomic_init(atomic<_ITp>* __a, __atomic_val_t<_ITp> __i) noexcept
{ __a->store(__i, memory_order_relaxed); }
template<typename _ITp>
inline void
atomic_init(volatile atomic<_ITp>* __a, __atomic_val_t<_ITp> __i) noexcept
{ __a->store(__i, memory_order_relaxed); }
template<typename _ITp>
inline void
atomic_store_explicit(atomic<_ITp>* __a, __atomic_val_t<_ITp> __i,
memory_order __m) noexcept
{ __a->store(__i, __m); }
template<typename _ITp>
inline void
atomic_store_explicit(volatile atomic<_ITp>* __a, __atomic_val_t<_ITp> __i,
memory_order __m) noexcept
{ __a->store(__i, __m); }
template<typename _ITp>
inline _ITp
atomic_load_explicit(const atomic<_ITp>* __a, memory_order __m) noexcept
{ return __a->load(__m); }
template<typename _ITp>
inline _ITp
atomic_load_explicit(const volatile atomic<_ITp>* __a,
memory_order __m) noexcept
{ return __a->load(__m); }
template<typename _ITp>
inline _ITp
atomic_exchange_explicit(atomic<_ITp>* __a, __atomic_val_t<_ITp> __i,
memory_order __m) noexcept
{ return __a->exchange(__i, __m); }
template<typename _ITp>
inline _ITp
atomic_exchange_explicit(volatile atomic<_ITp>* __a,
__atomic_val_t<_ITp> __i,
memory_order __m) noexcept
{ return __a->exchange(__i, __m); }
template<typename _ITp>
inline bool
atomic_compare_exchange_weak_explicit(atomic<_ITp>* __a,
__atomic_val_t<_ITp>* __i1,
__atomic_val_t<_ITp> __i2,
memory_order __m1,
memory_order __m2) noexcept
{ return __a->compare_exchange_weak(*__i1, __i2, __m1, __m2); }
template<typename _ITp>
inline bool
atomic_compare_exchange_weak_explicit(volatile atomic<_ITp>* __a,
__atomic_val_t<_ITp>* __i1,
__atomic_val_t<_ITp> __i2,
memory_order __m1,
memory_order __m2) noexcept
{ return __a->compare_exchange_weak(*__i1, __i2, __m1, __m2); }
template<typename _ITp>
inline bool
atomic_compare_exchange_strong_explicit(atomic<_ITp>* __a,
__atomic_val_t<_ITp>* __i1,
__atomic_val_t<_ITp> __i2,
memory_order __m1,
memory_order __m2) noexcept
{ return __a->compare_exchange_strong(*__i1, __i2, __m1, __m2); }
template<typename _ITp>
inline bool
atomic_compare_exchange_strong_explicit(volatile atomic<_ITp>* __a,
__atomic_val_t<_ITp>* __i1,
__atomic_val_t<_ITp> __i2,
memory_order __m1,
memory_order __m2) noexcept
{ return __a->compare_exchange_strong(*__i1, __i2, __m1, __m2); }
template<typename _ITp>
inline void
atomic_store(atomic<_ITp>* __a, __atomic_val_t<_ITp> __i) noexcept
{ atomic_store_explicit(__a, __i, memory_order_seq_cst); }
template<typename _ITp>
inline void
atomic_store(volatile atomic<_ITp>* __a, __atomic_val_t<_ITp> __i) noexcept
{ atomic_store_explicit(__a, __i, memory_order_seq_cst); }
template<typename _ITp>
inline _ITp
atomic_load(const atomic<_ITp>* __a) noexcept
{ return atomic_load_explicit(__a, memory_order_seq_cst); }
template<typename _ITp>
inline _ITp
atomic_load(const volatile atomic<_ITp>* __a) noexcept
{ return atomic_load_explicit(__a, memory_order_seq_cst); }
template<typename _ITp>
inline _ITp
atomic_exchange(atomic<_ITp>* __a, __atomic_val_t<_ITp> __i) noexcept
{ return atomic_exchange_explicit(__a, __i, memory_order_seq_cst); }
template<typename _ITp>
inline _ITp
atomic_exchange(volatile atomic<_ITp>* __a,
__atomic_val_t<_ITp> __i) noexcept
{ return atomic_exchange_explicit(__a, __i, memory_order_seq_cst); }
template<typename _ITp>
inline bool
atomic_compare_exchange_weak(atomic<_ITp>* __a,
__atomic_val_t<_ITp>* __i1,
__atomic_val_t<_ITp> __i2) noexcept
{
return atomic_compare_exchange_weak_explicit(__a, __i1, __i2,
memory_order_seq_cst,
memory_order_seq_cst);
}
template<typename _ITp>
inline bool
atomic_compare_exchange_weak(volatile atomic<_ITp>* __a,
__atomic_val_t<_ITp>* __i1,
__atomic_val_t<_ITp> __i2) noexcept
{
return atomic_compare_exchange_weak_explicit(__a, __i1, __i2,
memory_order_seq_cst,
memory_order_seq_cst);
}
template<typename _ITp>
inline bool
atomic_compare_exchange_strong(atomic<_ITp>* __a,
__atomic_val_t<_ITp>* __i1,
__atomic_val_t<_ITp> __i2) noexcept
{
return atomic_compare_exchange_strong_explicit(__a, __i1, __i2,
memory_order_seq_cst,
memory_order_seq_cst);
}
template<typename _ITp>
inline bool
atomic_compare_exchange_strong(volatile atomic<_ITp>* __a,
__atomic_val_t<_ITp>* __i1,
__atomic_val_t<_ITp> __i2) noexcept
{
return atomic_compare_exchange_strong_explicit(__a, __i1, __i2,
memory_order_seq_cst,
memory_order_seq_cst);
}
template<typename _ITp>
inline _ITp
atomic_fetch_add_explicit(atomic<_ITp>* __a,
__atomic_diff_t<_ITp> __i,
memory_order __m) noexcept
{ return __a->fetch_add(__i, __m); }
template<typename _ITp>
inline _ITp
atomic_fetch_add_explicit(volatile atomic<_ITp>* __a,
__atomic_diff_t<_ITp> __i,
memory_order __m) noexcept
{ return __a->fetch_add(__i, __m); }
template<typename _ITp>
inline _ITp
atomic_fetch_sub_explicit(atomic<_ITp>* __a,
__atomic_diff_t<_ITp> __i,
memory_order __m) noexcept
{ return __a->fetch_sub(__i, __m); }
template<typename _ITp>
inline _ITp
atomic_fetch_sub_explicit(volatile atomic<_ITp>* __a,
__atomic_diff_t<_ITp> __i,
memory_order __m) noexcept
{ return __a->fetch_sub(__i, __m); }
template<typename _ITp>
inline _ITp
atomic_fetch_and_explicit(__atomic_base<_ITp>* __a,
__atomic_val_t<_ITp> __i,
memory_order __m) noexcept
{ return __a->fetch_and(__i, __m); }
template<typename _ITp>
inline _ITp
atomic_fetch_and_explicit(volatile __atomic_base<_ITp>* __a,
__atomic_val_t<_ITp> __i,
memory_order __m) noexcept
{ return __a->fetch_and(__i, __m); }
template<typename _ITp>
inline _ITp
atomic_fetch_or_explicit(__atomic_base<_ITp>* __a,
__atomic_val_t<_ITp> __i,
memory_order __m) noexcept
{ return __a->fetch_or(__i, __m); }
template<typename _ITp>
inline _ITp
atomic_fetch_or_explicit(volatile __atomic_base<_ITp>* __a,
__atomic_val_t<_ITp> __i,
memory_order __m) noexcept
{ return __a->fetch_or(__i, __m); }
template<typename _ITp>
inline _ITp
atomic_fetch_xor_explicit(__atomic_base<_ITp>* __a,
__atomic_val_t<_ITp> __i,
memory_order __m) noexcept
{ return __a->fetch_xor(__i, __m); }
template<typename _ITp>
inline _ITp
atomic_fetch_xor_explicit(volatile __atomic_base<_ITp>* __a,
__atomic_val_t<_ITp> __i,
memory_order __m) noexcept
{ return __a->fetch_xor(__i, __m); }
template<typename _ITp>
inline _ITp
atomic_fetch_add(atomic<_ITp>* __a,
__atomic_diff_t<_ITp> __i) noexcept
{ return atomic_fetch_add_explicit(__a, __i, memory_order_seq_cst); }
template<typename _ITp>
inline _ITp
atomic_fetch_add(volatile atomic<_ITp>* __a,
__atomic_diff_t<_ITp> __i) noexcept
{ return atomic_fetch_add_explicit(__a, __i, memory_order_seq_cst); }
template<typename _ITp>
inline _ITp
atomic_fetch_sub(atomic<_ITp>* __a,
__atomic_diff_t<_ITp> __i) noexcept
{ return atomic_fetch_sub_explicit(__a, __i, memory_order_seq_cst); }
template<typename _ITp>
inline _ITp
atomic_fetch_sub(volatile atomic<_ITp>* __a,
__atomic_diff_t<_ITp> __i) noexcept
{ return atomic_fetch_sub_explicit(__a, __i, memory_order_seq_cst); }
template<typename _ITp>
inline _ITp
atomic_fetch_and(__atomic_base<_ITp>* __a,
__atomic_val_t<_ITp> __i) noexcept
{ return atomic_fetch_and_explicit(__a, __i, memory_order_seq_cst); }
template<typename _ITp>
inline _ITp
atomic_fetch_and(volatile __atomic_base<_ITp>* __a,
__atomic_val_t<_ITp> __i) noexcept
{ return atomic_fetch_and_explicit(__a, __i, memory_order_seq_cst); }
template<typename _ITp>
inline _ITp
atomic_fetch_or(__atomic_base<_ITp>* __a,
__atomic_val_t<_ITp> __i) noexcept
{ return atomic_fetch_or_explicit(__a, __i, memory_order_seq_cst); }
template<typename _ITp>
inline _ITp
atomic_fetch_or(volatile __atomic_base<_ITp>* __a,
__atomic_val_t<_ITp> __i) noexcept
{ return atomic_fetch_or_explicit(__a, __i, memory_order_seq_cst); }
template<typename _ITp>
inline _ITp
atomic_fetch_xor(__atomic_base<_ITp>* __a,
__atomic_val_t<_ITp> __i) noexcept
{ return atomic_fetch_xor_explicit(__a, __i, memory_order_seq_cst); }
template<typename _ITp>
inline _ITp
atomic_fetch_xor(volatile __atomic_base<_ITp>* __a,
__atomic_val_t<_ITp> __i) noexcept
{ return atomic_fetch_xor_explicit(__a, __i, memory_order_seq_cst); }
}
namespace hwy {
static constexpr __attribute__((unused)) size_t kMaxVectorSize = 64;
#pragma pack(push, 1)
struct float16_t {
uint16_t bits;
};
struct bfloat16_t {
uint16_t bits;
};
#pragma pack(pop)
using float32_t = float;
using float64_t = double;
#pragma pack(push, 1)
struct alignas(16) uint128_t {
uint64_t lo;
uint64_t hi;
};
struct alignas(16) K64V64 {
uint64_t value;
uint64_t key;
};
struct alignas(8) K32V32 {
uint32_t value;
uint32_t key;
};
#pragma pack(pop)
static inline __attribute__((unused)) bool operator<(const uint128_t& a,
const uint128_t& b) {
return (a.hi == b.hi) ? a.lo < b.lo : a.hi < b.hi;
}
static inline __attribute__((unused)) bool operator>(const uint128_t& a,
const uint128_t& b) {
return b < a;
}
static inline __attribute__((unused)) bool operator==(const uint128_t& a,
const uint128_t& b) {
return a.lo == b.lo && a.hi == b.hi;
}
static inline __attribute__((unused)) bool operator<(const K64V64& a,
const K64V64& b) {
return a.key < b.key;
}
static inline __attribute__((unused)) bool operator>(const K64V64& a,
const K64V64& b) {
return b < a;
}
static inline __attribute__((unused)) bool operator==(const K64V64& a,
const K64V64& b) {
return a.key == b.key;
}
static inline __attribute__((unused)) bool operator<(const K32V32& a,
const K32V32& b) {
return a.key < b.key;
}
static inline __attribute__((unused)) bool operator>(const K32V32& a,
const K32V32& b) {
return b < a;
}
static inline __attribute__((unused)) bool operator==(const K32V32& a,
const K32V32& b) {
return a.key == b.key;
}
template <bool Condition>
struct EnableIfT {};
template <>
struct EnableIfT<true> {
using type = void;
};
template <bool Condition>
using EnableIf = typename EnableIfT<Condition>::type;
template <typename T, typename U>
struct IsSameT {
enum { value = 0 };
};
template <typename T>
struct IsSameT<T, T> {
enum { value = 1 };
};
template <typename T, typename U>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) constexpr bool IsSame() {
return IsSameT<T, U>::value;
}
template <bool Condition, typename Then, typename Else>
struct IfT {
using type = Then;
};
template <class Then, class Else>
struct IfT<false, Then, Else> {
using type = Else;
};
template <bool Condition, typename Then, typename Else>
using If = typename IfT<Condition, Then, Else>::type;
template <size_t N>
struct SizeTag {};
template <class T>
struct RemoveConstT {
using type = T;
};
template <class T>
struct RemoveConstT<const T> {
using type = T;
};
template <class T>
using RemoveConst = typename RemoveConstT<T>::type;
template <class T>
struct RemoveRefT {
using type = T;
};
template <class T>
struct RemoveRefT<T&> {
using type = T;
};
template <class T>
using RemoveRef = typename RemoveRefT<T>::type;
namespace detail {
template <typename T>
struct Relations;
template <>
struct Relations<uint8_t> {
using Unsigned = uint8_t;
using Signed = int8_t;
using Wide = uint16_t;
enum { is_signed = 0, is_float = 0 };
};
template <>
struct Relations<int8_t> {
using Unsigned = uint8_t;
using Signed = int8_t;
using Wide = int16_t;
enum { is_signed = 1, is_float = 0 };
};
template <>
struct Relations<uint16_t> {
using Unsigned = uint16_t;
using Signed = int16_t;
using Wide = uint32_t;
using Narrow = uint8_t;
enum { is_signed = 0, is_float = 0 };
};
template <>
struct Relations<int16_t> {
using Unsigned = uint16_t;
using Signed = int16_t;
using Wide = int32_t;
using Narrow = int8_t;
enum { is_signed = 1, is_float = 0 };
};
template <>
struct Relations<uint32_t> {
using Unsigned = uint32_t;
using Signed = int32_t;
using Float = float;
using Wide = uint64_t;
using Narrow = uint16_t;
enum { is_signed = 0, is_float = 0 };
};
template <>
struct Relations<int32_t> {
using Unsigned = uint32_t;
using Signed = int32_t;
using Float = float;
using Wide = int64_t;
using Narrow = int16_t;
enum { is_signed = 1, is_float = 0 };
};
template <>
struct Relations<uint64_t> {
using Unsigned = uint64_t;
using Signed = int64_t;
using Float = double;
using Wide = uint128_t;
using Narrow = uint32_t;
enum { is_signed = 0, is_float = 0 };
};
template <>
struct Relations<int64_t> {
using Unsigned = uint64_t;
using Signed = int64_t;
using Float = double;
using Narrow = int32_t;
enum { is_signed = 1, is_float = 0 };
};
template <>
struct Relations<uint128_t> {
using Unsigned = uint128_t;
using Narrow = uint64_t;
enum { is_signed = 0, is_float = 0 };
};
template <>
struct Relations<float16_t> {
using Unsigned = uint16_t;
using Signed = int16_t;
using Float = float16_t;
using Wide = float;
enum { is_signed = 1, is_float = 1 };
};
template <>
struct Relations<bfloat16_t> {
using Unsigned = uint16_t;
using Signed = int16_t;
using Wide = float;
enum { is_signed = 1, is_float = 1 };
};
template <>
struct Relations<float> {
using Unsigned = uint32_t;
using Signed = int32_t;
using Float = float;
using Wide = double;
using Narrow = float16_t;
enum { is_signed = 1, is_float = 1 };
};
template <>
struct Relations<double> {
using Unsigned = uint64_t;
using Signed = int64_t;
using Float = double;
using Narrow = float;
enum { is_signed = 1, is_float = 1 };
};
template <size_t N>
struct TypeFromSize;
template <>
struct TypeFromSize<1> {
using Unsigned = uint8_t;
using Signed = int8_t;
};
template <>
struct TypeFromSize<2> {
using Unsigned = uint16_t;
using Signed = int16_t;
};
template <>
struct TypeFromSize<4> {
using Unsigned = uint32_t;
using Signed = int32_t;
using Float = float;
};
template <>
struct TypeFromSize<8> {
using Unsigned = uint64_t;
using Signed = int64_t;
using Float = double;
};
template <>
struct TypeFromSize<16> {
using Unsigned = uint128_t;
};
}
template <typename T>
using MakeUnsigned = typename detail::Relations<T>::Unsigned;
template <typename T>
using MakeSigned = typename detail::Relations<T>::Signed;
template <typename T>
using MakeFloat = typename detail::Relations<T>::Float;
template <typename T>
using MakeWide = typename detail::Relations<T>::Wide;
template <typename T>
using MakeNarrow = typename detail::Relations<T>::Narrow;
template <size_t N>
using UnsignedFromSize = typename detail::TypeFromSize<N>::Unsigned;
template <size_t N>
using SignedFromSize = typename detail::TypeFromSize<N>::Signed;
template <size_t N>
using FloatFromSize = typename detail::TypeFromSize<N>::Float;
using UnsignedTag = SizeTag<0>;
using SignedTag = SizeTag<0x100>;
using FloatTag = SizeTag<0x200>;
template <typename T, class R = detail::Relations<T>>
constexpr auto TypeTag() -> hwy::SizeTag<((R::is_signed + R::is_float) << 8)> {
return hwy::SizeTag<((R::is_signed + R::is_float) << 8)>();
}
using NonFloatTag = SizeTag<0x400>;
template <typename T, class R = detail::Relations<T>>
constexpr auto IsFloatTag() -> hwy::SizeTag<(R::is_float ? 0x200 : 0x400)> {
return hwy::SizeTag<(R::is_float ? 0x200 : 0x400)>();
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) constexpr bool IsFloat() {
return IsSame<T, float>() || IsSame<T, double>();
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) constexpr bool IsSpecialFloat() {
return IsSame<T, float16_t>() || IsSame<T, bfloat16_t>();
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) constexpr bool IsSigned() {
return T(0) > T(-1);
}
template <>
constexpr bool IsSigned<float16_t>() {
return true;
}
template <>
constexpr bool IsSigned<bfloat16_t>() {
return true;
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) constexpr T LimitsMax() {
static_assert(!IsFloat<T>(), "Only for integer types");
using TU = MakeUnsigned<T>;
return static_cast<T>(IsSigned<T>() ? (static_cast<TU>(~0ull) >> 1)
: static_cast<TU>(~0ull));
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) constexpr T LimitsMin() {
static_assert(!IsFloat<T>(), "Only for integer types");
return IsSigned<T>() ? T(-1) - LimitsMax<T>() : T(0);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) constexpr T LowestValue() {
return LimitsMin<T>();
}
template <>
constexpr float LowestValue<float>() {
return -3.402823466e+38F;
}
template <>
constexpr double LowestValue<double>() {
return -1.7976931348623158e+308;
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) constexpr T HighestValue() {
return LimitsMax<T>();
}
template <>
constexpr float HighestValue<float>() {
return 3.402823466e+38F;
}
template <>
constexpr double HighestValue<double>() {
return 1.7976931348623158e+308;
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) constexpr T Epsilon() {
return 1;
}
template <>
constexpr float Epsilon<float>() {
return 1.192092896e-7f;
}
template <>
constexpr double Epsilon<double>() {
return 2.2204460492503131e-16;
}
template <typename T>
constexpr int MantissaBits() {
static_assert(sizeof(T) == 0, "Only instantiate the specializations");
return 0;
}
template <>
constexpr int MantissaBits<float>() {
return 23;
}
template <>
constexpr int MantissaBits<double>() {
return 52;
}
template <typename T>
constexpr MakeSigned<T> MaxExponentTimes2() {
return -(MakeSigned<T>{1} << (MantissaBits<T>() + 1));
}
template <typename T>
constexpr MakeUnsigned<T> SignMask() {
return MakeUnsigned<T>{1} << (sizeof(T) * 8 - 1);
}
template <typename T>
constexpr MakeUnsigned<T> ExponentMask() {
return (~(MakeUnsigned<T>{1} << MantissaBits<T>()) + 1) & ~SignMask<T>();
}
template <typename T>
constexpr MakeUnsigned<T> MantissaMask() {
return (MakeUnsigned<T>{1} << MantissaBits<T>()) - 1;
}
template <typename T>
constexpr T MantissaEnd() {
static_assert(sizeof(T) == 0, "Only instantiate the specializations");
return 0;
}
template <>
constexpr float MantissaEnd<float>() {
return 8388608.0f;
}
template <>
constexpr double MantissaEnd<double>() {
return 4503599627370496.0;
}
template <typename T>
constexpr int ExponentBits() {
return 8 * sizeof(T) - 1 - MantissaBits<T>();
}
template <typename T>
constexpr MakeSigned<T> MaxExponentField() {
return (MakeSigned<T>{1} << ExponentBits<T>()) - 1;
}
template <typename T1, typename T2>
constexpr inline T1 DivCeil(T1 a, T2 b) {
return (a + b - 1) / b;
}
constexpr inline size_t RoundUpTo(size_t what, size_t align) {
return DivCeil(what, align) * align;
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t Num0BitsBelowLS1Bit_Nonzero32(const uint32_t x) {
return static_cast<size_t>(__builtin_ctz(x));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t Num0BitsBelowLS1Bit_Nonzero64(const uint64_t x) {
return static_cast<size_t>(__builtin_ctzll(x));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t Num0BitsAboveMS1Bit_Nonzero32(const uint32_t x) {
return static_cast<size_t>(__builtin_clz(x));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t Num0BitsAboveMS1Bit_Nonzero64(const uint64_t x) {
return static_cast<size_t>(__builtin_clzll(x));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t PopCount(uint64_t x) {
return static_cast<size_t>(__builtin_popcountll(x));
}
template <typename TI>
constexpr size_t FloorLog2(TI x) {
return x == TI{1}
? 0
: static_cast<size_t>(FloorLog2(static_cast<TI>(x >> 1)) + 1);
}
template <typename TI>
constexpr size_t CeilLog2(TI x) {
return x == TI{1}
? 0
: static_cast<size_t>(FloorLog2(static_cast<TI>(x - 1)) + 1);
}
template <typename T>
inline __attribute__((always_inline)) constexpr T AddWithWraparound(hwy::FloatTag , T t, size_t n) {
return t + static_cast<T>(n);
}
template <typename T>
inline __attribute__((always_inline)) constexpr T AddWithWraparound(hwy::NonFloatTag , T t,
size_t n) {
using TU = MakeUnsigned<T>;
return static_cast<T>(
static_cast<TU>(static_cast<TU>(t) + static_cast<TU>(n)) &
hwy::LimitsMax<TU>());
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) uint64_t Mul128(uint64_t a, uint64_t b, uint64_t* __restrict__ upper) {
constexpr uint64_t kLo32 = 0xFFFFFFFFU;
const uint64_t lo_lo = (a & kLo32) * (b & kLo32);
const uint64_t hi_lo = (a >> 32) * (b & kLo32);
const uint64_t lo_hi = (a & kLo32) * (b >> 32);
const uint64_t hi_hi = (a >> 32) * (b >> 32);
const uint64_t t = (lo_lo >> 32) + (hi_lo & kLo32) + lo_hi;
*upper = (hi_lo >> 32) + (t >> 32) + hi_hi;
return (t << 32) | (lo_lo & kLo32);
}
template <size_t kBytes, typename From, typename To>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void CopyBytes(const From* from, To* to) {
__builtin_memcpy(
static_cast<void*>(to), static_cast<const void*>(from), kBytes);
}
template <typename From, typename To>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void CopySameSize(const From* __restrict__ from, To* __restrict__ to) {
static_assert(sizeof(From) == sizeof(To), "");
CopyBytes<sizeof(From)>(from, to);
}
template <size_t kBytes, typename To>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void ZeroBytes(To* to) {
__builtin_memset(to, 0, kBytes);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) float F32FromBF16(bfloat16_t bf) {
uint32_t bits = bf.bits;
bits <<= 16;
float f;
CopySameSize(&bits, &f);
return f;
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) bfloat16_t BF16FromF32(float f) {
uint32_t bits;
CopySameSize(&f, &bits);
bfloat16_t bf;
bf.bits = static_cast<uint16_t>(bits >> 16);
return bf;
}
__attribute__((noreturn)) void __attribute__((__format__(__printf__, 3, 4)))
Abort(const char* file, int line, const char* format, ...);
template <class T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void PreventElision(T&& output) {
asm volatile("" : "+r"(output) : : "memory");
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Tp>
class __new_allocator
{
public:
typedef _Tp value_type;
typedef std::size_t size_type;
typedef std::ptrdiff_t difference_type;
typedef _Tp* pointer;
typedef const _Tp* const_pointer;
typedef _Tp& reference;
typedef const _Tp& const_reference;
template<typename _Tp1>
struct rebind
{ typedef __new_allocator<_Tp1> other; };
typedef std::true_type propagate_on_container_move_assignment;
__attribute__((__always_inline__))
__new_allocator() noexcept { }
__attribute__((__always_inline__))
__new_allocator(const __new_allocator&) noexcept { }
template<typename _Tp1>
__attribute__((__always_inline__))
__new_allocator(const __new_allocator<_Tp1>&) noexcept { }
~__new_allocator() noexcept { }
pointer
address(reference __x) const noexcept
{ return std::__addressof(__x); }
const_pointer
address(const_reference __x) const noexcept
{ return std::__addressof(__x); }
[[__nodiscard__]] _Tp*
allocate(size_type __n, const void* = static_cast<const void*>(0))
{
static_assert(sizeof(_Tp) != 0, "cannot allocate incomplete types");
if (__builtin_expect(__n > this->_M_max_size(), false))
{
if (__n > (std::size_t(-1) / sizeof(_Tp)))
std::__throw_bad_array_new_length();
std::__throw_bad_alloc();
}
if (alignof(_Tp) > 16)
{
std::align_val_t __al = std::align_val_t(alignof(_Tp));
return static_cast<_Tp*>(::operator new(__n * sizeof(_Tp),
__al));
}
return static_cast<_Tp*>(::operator new(__n * sizeof(_Tp)));
}
void
deallocate(_Tp* __p, size_type __n __attribute__ ((__unused__)))
{
if (alignof(_Tp) > 16)
{
::operator delete((__p), (__n) * sizeof(_Tp),
std::align_val_t(alignof(_Tp)));
return;
}
::operator delete((__p), (__n) * sizeof(_Tp));
}
__attribute__((__always_inline__))
size_type
max_size() const noexcept
{ return _M_max_size(); }
template<typename _Up, typename... _Args>
__attribute__((__always_inline__))
void
construct(_Up* __p, _Args&&... __args)
noexcept(std::is_nothrow_constructible<_Up, _Args...>::value)
{ ::new((void *)__p) _Up(std::forward<_Args>(__args)...); }
template<typename _Up>
__attribute__((__always_inline__))
void
destroy(_Up* __p)
noexcept(std::is_nothrow_destructible<_Up>::value)
{ __p->~_Up(); }
template<typename _Up>
friend __attribute__((__always_inline__)) bool
operator==(const __new_allocator&, const __new_allocator<_Up>&)
noexcept
{ return true; }
template<typename _Up>
friend __attribute__((__always_inline__)) bool
operator!=(const __new_allocator&, const __new_allocator<_Up>&)
noexcept
{ return false; }
private:
__attribute__((__always_inline__))
constexpr size_type
_M_max_size() const noexcept
{
return std::size_t(0x7fffffff) / sizeof(_Tp);
}
};
}
namespace std
{
template<typename _Tp>
using __allocator_base = __new_allocator<_Tp>;
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename>
class allocator;
template<>
class allocator<void>;
template<typename, typename>
struct uses_allocator;
template<typename>
struct allocator_traits;
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<>
class allocator<void>
{
public:
typedef void value_type;
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef void* pointer;
typedef const void* const_pointer;
template<typename _Tp1>
struct rebind
{ typedef allocator<_Tp1> other; };
using propagate_on_container_move_assignment = true_type;
using is_always_equal
= true_type;
};
template<typename _Tp>
class allocator : public __allocator_base<_Tp>
{
public:
typedef _Tp value_type;
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef _Tp* pointer;
typedef const _Tp* const_pointer;
typedef _Tp& reference;
typedef const _Tp& const_reference;
template<typename _Tp1>
struct rebind
{ typedef allocator<_Tp1> other; };
using propagate_on_container_move_assignment = true_type;
using is_always_equal
= true_type;
__attribute__((__always_inline__))
allocator() noexcept { }
__attribute__((__always_inline__))
allocator(const allocator& __a) noexcept
: __allocator_base<_Tp>(__a) { }
allocator& operator=(const allocator&) = default;
template<typename _Tp1>
__attribute__((__always_inline__))
allocator(const allocator<_Tp1>&) noexcept { }
__attribute__((__always_inline__))
~allocator() noexcept { }
friend __attribute__((__always_inline__))
bool
operator==(const allocator&, const allocator&) noexcept
{ return true; }
friend __attribute__((__always_inline__))
bool
operator!=(const allocator&, const allocator&) noexcept
{ return false; }
};
template<typename _T1, typename _T2>
__attribute__((__always_inline__))
inline bool
operator==(const allocator<_T1>&, const allocator<_T2>&)
noexcept
{ return true; }
template<typename _T1, typename _T2>
__attribute__((__always_inline__))
inline bool
operator!=(const allocator<_T1>&, const allocator<_T2>&)
noexcept
{ return false; }
template<typename _Tp>
class allocator<const _Tp>
{
public:
typedef _Tp value_type;
template<typename _Up> allocator(const allocator<_Up>&) { }
};
template<typename _Tp>
class allocator<volatile _Tp>
{
public:
typedef _Tp value_type;
template<typename _Up> allocator(const allocator<_Up>&) { }
};
template<typename _Tp>
class allocator<const volatile _Tp>
{
public:
typedef _Tp value_type;
template<typename _Up> allocator(const allocator<_Up>&) { }
};
extern template class allocator<char>;
extern template class allocator<wchar_t>;
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template <typename _Tp>
inline void
destroy_at(_Tp* __location)
{
if constexpr (201703L > 201703L && is_array_v<_Tp>)
{
for (auto& __x : *__location)
std::destroy_at(std::__addressof(__x));
}
else
__location->~_Tp();
}
template<typename _Tp, typename... _Args>
inline void
_Construct(_Tp* __p, _Args&&... __args)
{
::new((void*)__p) _Tp(std::forward<_Args>(__args)...);
}
template<typename _T1>
inline void
_Construct_novalue(_T1* __p)
{ ::new((void*)__p) _T1; }
template<typename _ForwardIterator>
void
_Destroy(_ForwardIterator __first, _ForwardIterator __last);
template<typename _Tp>
constexpr inline void
_Destroy(_Tp* __pointer)
{
__pointer->~_Tp();
}
template<bool>
struct _Destroy_aux
{
template<typename _ForwardIterator>
static void
__destroy(_ForwardIterator __first, _ForwardIterator __last)
{
for (; __first != __last; ++__first)
std::_Destroy(std::__addressof(*__first));
}
};
template<>
struct _Destroy_aux<true>
{
template<typename _ForwardIterator>
static void
__destroy(_ForwardIterator, _ForwardIterator) { }
};
template<typename _ForwardIterator>
inline void
_Destroy(_ForwardIterator __first, _ForwardIterator __last)
{
typedef typename iterator_traits<_ForwardIterator>::value_type
_Value_type;
static_assert(is_destructible<_Value_type>::value,
"value type is destructible");
std::_Destroy_aux<__has_trivial_destructor(_Value_type)>::
__destroy(__first, __last);
}
template<bool>
struct _Destroy_n_aux
{
template<typename _ForwardIterator, typename _Size>
static _ForwardIterator
__destroy_n(_ForwardIterator __first, _Size __count)
{
for (; __count > 0; (void)++__first, --__count)
std::_Destroy(std::__addressof(*__first));
return __first;
}
};
template<>
struct _Destroy_n_aux<true>
{
template<typename _ForwardIterator, typename _Size>
static _ForwardIterator
__destroy_n(_ForwardIterator __first, _Size __count)
{
std::advance(__first, __count);
return __first;
}
};
template<typename _ForwardIterator, typename _Size>
inline _ForwardIterator
_Destroy_n(_ForwardIterator __first, _Size __count)
{
typedef typename iterator_traits<_ForwardIterator>::value_type
_Value_type;
static_assert(is_destructible<_Value_type>::value,
"value type is destructible");
return std::_Destroy_n_aux<__has_trivial_destructor(_Value_type)>::
__destroy_n(__first, __count);
}
template <typename _ForwardIterator>
inline void
destroy(_ForwardIterator __first, _ForwardIterator __last)
{
std::_Destroy(__first, __last);
}
template <typename _ForwardIterator, typename _Size>
inline _ForwardIterator
destroy_n(_ForwardIterator __first, _Size __count)
{
return std::_Destroy_n(__first, __count);
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
struct __allocator_traits_base
{
template<typename _Tp, typename _Up, typename = void>
struct __rebind : __replace_first_arg<_Tp, _Up>
{
static_assert(is_same<
typename __replace_first_arg<_Tp, typename _Tp::value_type>::type,
_Tp>::value,
"allocator_traits<A>::rebind_alloc<A::value_type> must be A");
};
template<typename _Tp, typename _Up>
struct __rebind<_Tp, _Up,
__void_t<typename _Tp::template rebind<_Up>::other>>
{
using type = typename _Tp::template rebind<_Up>::other;
static_assert(is_same<
typename _Tp::template rebind<typename _Tp::value_type>::other,
_Tp>::value,
"allocator_traits<A>::rebind_alloc<A::value_type> must be A");
};
protected:
template<typename _Tp>
using __pointer = typename _Tp::pointer;
template<typename _Tp>
using __c_pointer = typename _Tp::const_pointer;
template<typename _Tp>
using __v_pointer = typename _Tp::void_pointer;
template<typename _Tp>
using __cv_pointer = typename _Tp::const_void_pointer;
template<typename _Tp>
using __pocca = typename _Tp::propagate_on_container_copy_assignment;
template<typename _Tp>
using __pocma = typename _Tp::propagate_on_container_move_assignment;
template<typename _Tp>
using __pocs = typename _Tp::propagate_on_container_swap;
template<typename _Tp>
using __equal = __type_identity<typename _Tp::is_always_equal>;
};
template<typename _Alloc, typename _Up>
using __alloc_rebind
= typename __allocator_traits_base::template __rebind<_Alloc, _Up>::type;
template<typename _Alloc>
struct allocator_traits : __allocator_traits_base
{
typedef _Alloc allocator_type;
typedef typename _Alloc::value_type value_type;
using pointer = __detected_or_t<value_type*, __pointer, _Alloc>;
private:
template<template<typename> class _Func, typename _Tp, typename = void>
struct _Ptr
{
using type = typename pointer_traits<pointer>::template rebind<_Tp>;
};
template<template<typename> class _Func, typename _Tp>
struct _Ptr<_Func, _Tp, __void_t<_Func<_Alloc>>>
{
using type = _Func<_Alloc>;
};
template<typename _A2, typename _PtrT, typename = void>
struct _Diff
{ using type = typename pointer_traits<_PtrT>::difference_type; };
template<typename _A2, typename _PtrT>
struct _Diff<_A2, _PtrT, __void_t<typename _A2::difference_type>>
{ using type = typename _A2::difference_type; };
template<typename _A2, typename _DiffT, typename = void>
struct _Size : make_unsigned<_DiffT> { };
template<typename _A2, typename _DiffT>
struct _Size<_A2, _DiffT, __void_t<typename _A2::size_type>>
{ using type = typename _A2::size_type; };
public:
using const_pointer = typename _Ptr<__c_pointer, const value_type>::type;
using void_pointer = typename _Ptr<__v_pointer, void>::type;
using const_void_pointer = typename _Ptr<__cv_pointer, const void>::type;
using difference_type = typename _Diff<_Alloc, pointer>::type;
using size_type = typename _Size<_Alloc, difference_type>::type;
using propagate_on_container_copy_assignment
= __detected_or_t<false_type, __pocca, _Alloc>;
using propagate_on_container_move_assignment
= __detected_or_t<false_type, __pocma, _Alloc>;
using propagate_on_container_swap
= __detected_or_t<false_type, __pocs, _Alloc>;
using is_always_equal
= typename __detected_or_t<is_empty<_Alloc>, __equal, _Alloc>::type;
template<typename _Tp>
using rebind_alloc = __alloc_rebind<_Alloc, _Tp>;
template<typename _Tp>
using rebind_traits = allocator_traits<rebind_alloc<_Tp>>;
private:
template<typename _Alloc2>
static constexpr auto
_S_allocate(_Alloc2& __a, size_type __n, const_void_pointer __hint, int)
-> decltype(__a.allocate(__n, __hint))
{ return __a.allocate(__n, __hint); }
template<typename _Alloc2>
static constexpr pointer
_S_allocate(_Alloc2& __a, size_type __n, const_void_pointer, ...)
{ return __a.allocate(__n); }
template<typename _Tp, typename... _Args>
struct __construct_helper
{
template<typename _Alloc2,
typename = decltype(std::declval<_Alloc2*>()->construct(
std::declval<_Tp*>(), std::declval<_Args>()...))>
static true_type __test(int);
template<typename>
static false_type __test(...);
using type = decltype(__test<_Alloc>(0));
};
template<typename _Tp, typename... _Args>
using __has_construct
= typename __construct_helper<_Tp, _Args...>::type;
template<typename _Tp, typename... _Args>
static constexpr _Require<__has_construct<_Tp, _Args...>>
_S_construct(_Alloc& __a, _Tp* __p, _Args&&... __args)
noexcept(noexcept(__a.construct(__p, std::forward<_Args>(__args)...)))
{ __a.construct(__p, std::forward<_Args>(__args)...); }
template<typename _Tp, typename... _Args>
static constexpr
_Require<__and_<__not_<__has_construct<_Tp, _Args...>>,
is_constructible<_Tp, _Args...>>>
_S_construct(_Alloc&, _Tp* __p, _Args&&... __args)
noexcept(std::is_nothrow_constructible<_Tp, _Args...>::value)
{
::new((void*)__p) _Tp(std::forward<_Args>(__args)...);
}
template<typename _Alloc2, typename _Tp>
static constexpr auto
_S_destroy(_Alloc2& __a, _Tp* __p, int)
noexcept(noexcept(__a.destroy(__p)))
-> decltype(__a.destroy(__p))
{ __a.destroy(__p); }
template<typename _Alloc2, typename _Tp>
static constexpr void
_S_destroy(_Alloc2&, _Tp* __p, ...)
noexcept(std::is_nothrow_destructible<_Tp>::value)
{ std::_Destroy(__p); }
template<typename _Alloc2>
static constexpr auto
_S_max_size(_Alloc2& __a, int)
-> decltype(__a.max_size())
{ return __a.max_size(); }
template<typename _Alloc2>
static constexpr size_type
_S_max_size(_Alloc2&, ...)
{
return __gnu_cxx::__numeric_traits<size_type>::__max
/ sizeof(value_type);
}
template<typename _Alloc2>
static constexpr auto
_S_select(_Alloc2& __a, int)
-> decltype(__a.select_on_container_copy_construction())
{ return __a.select_on_container_copy_construction(); }
template<typename _Alloc2>
static constexpr _Alloc2
_S_select(_Alloc2& __a, ...)
{ return __a; }
public:
[[__nodiscard__]] static pointer
allocate(_Alloc& __a, size_type __n)
{ return __a.allocate(__n); }
[[__nodiscard__]] static pointer
allocate(_Alloc& __a, size_type __n, const_void_pointer __hint)
{ return _S_allocate(__a, __n, __hint, 0); }
static void
deallocate(_Alloc& __a, pointer __p, size_type __n)
{ __a.deallocate(__p, __n); }
template<typename _Tp, typename... _Args>
static auto
construct(_Alloc& __a, _Tp* __p, _Args&&... __args)
noexcept(noexcept(_S_construct(__a, __p,
std::forward<_Args>(__args)...)))
-> decltype(_S_construct(__a, __p, std::forward<_Args>(__args)...))
{ _S_construct(__a, __p, std::forward<_Args>(__args)...); }
template<typename _Tp>
static void
destroy(_Alloc& __a, _Tp* __p)
noexcept(noexcept(_S_destroy(__a, __p, 0)))
{ _S_destroy(__a, __p, 0); }
static size_type
max_size(const _Alloc& __a) noexcept
{ return _S_max_size(__a, 0); }
static _Alloc
select_on_container_copy_construction(const _Alloc& __rhs)
{ return _S_select(__rhs, 0); }
};
template<typename _Tp>
struct allocator_traits<allocator<_Tp>>
{
using allocator_type = allocator<_Tp>;
using value_type = _Tp;
using pointer = _Tp*;
using const_pointer = const _Tp*;
using void_pointer = void*;
using const_void_pointer = const void*;
using difference_type = std::ptrdiff_t;
using size_type = std::size_t;
using propagate_on_container_copy_assignment = false_type;
using propagate_on_container_move_assignment = true_type;
using propagate_on_container_swap = false_type;
using is_always_equal = true_type;
template<typename _Up>
using rebind_alloc = allocator<_Up>;
template<typename _Up>
using rebind_traits = allocator_traits<allocator<_Up>>;
[[__nodiscard__,__gnu__::__always_inline__]]
static pointer
allocate(allocator_type& __a, size_type __n)
{ return __a.allocate(__n); }
[[__nodiscard__,__gnu__::__always_inline__]]
static pointer
allocate(allocator_type& __a, size_type __n, const_void_pointer __hint)
{
return __a.allocate(__n, __hint);
}
[[__gnu__::__always_inline__]]
static void
deallocate(allocator_type& __a, pointer __p, size_type __n)
{ __a.deallocate(__p, __n); }
template<typename _Up, typename... _Args>
[[__gnu__::__always_inline__]]
static void
construct(allocator_type& __a __attribute__((__unused__)), _Up* __p,
_Args&&... __args)
noexcept(std::is_nothrow_constructible<_Up, _Args...>::value)
{
__a.construct(__p, std::forward<_Args>(__args)...);
}
template<typename _Up>
[[__gnu__::__always_inline__]]
static void
destroy(allocator_type& __a __attribute__((__unused__)), _Up* __p)
noexcept(is_nothrow_destructible<_Up>::value)
{
__a.destroy(__p);
}
[[__gnu__::__always_inline__]]
static size_type
max_size(const allocator_type& __a __attribute__((__unused__))) noexcept
{
return __a.max_size();
}
[[__gnu__::__always_inline__]]
static allocator_type
select_on_container_copy_construction(const allocator_type& __rhs)
{ return __rhs; }
};
template<>
struct allocator_traits<allocator<void>>
{
using allocator_type = allocator<void>;
using value_type = void;
using pointer = void*;
using const_pointer = const void*;
using void_pointer = void*;
using const_void_pointer = const void*;
using difference_type = std::ptrdiff_t;
using size_type = std::size_t;
using propagate_on_container_copy_assignment = false_type;
using propagate_on_container_move_assignment = true_type;
using propagate_on_container_swap = false_type;
using is_always_equal = true_type;
template<typename _Up>
using rebind_alloc = allocator<_Up>;
template<typename _Up>
using rebind_traits = allocator_traits<allocator<_Up>>;
static void*
allocate(allocator_type&, size_type, const void* = nullptr) = delete;
static void
deallocate(allocator_type&, void*, size_type) = delete;
template<typename _Up, typename... _Args>
[[__gnu__::__always_inline__]]
static void
construct(allocator_type&, _Up* __p, _Args&&... __args)
noexcept(std::is_nothrow_constructible<_Up, _Args...>::value)
{ std::_Construct(__p, std::forward<_Args>(__args)...); }
template<typename _Up>
[[__gnu__::__always_inline__]]
static void
destroy(allocator_type&, _Up* __p)
noexcept(is_nothrow_destructible<_Up>::value)
{ std::_Destroy(__p); }
static size_type
max_size(const allocator_type&) = delete;
[[__gnu__::__always_inline__]]
static allocator_type
select_on_container_copy_construction(const allocator_type& __rhs)
{ return __rhs; }
};
template<typename _Alloc>
[[__gnu__::__always_inline__]]
constexpr inline void
__alloc_on_copy(_Alloc& __one, const _Alloc& __two)
{
using __traits = allocator_traits<_Alloc>;
using __pocca =
typename __traits::propagate_on_container_copy_assignment::type;
if constexpr (__pocca::value)
__one = __two;
}
template<typename _Alloc>
[[__gnu__::__always_inline__]]
constexpr _Alloc
__alloc_on_copy(const _Alloc& __a)
{
typedef allocator_traits<_Alloc> __traits;
return __traits::select_on_container_copy_construction(__a);
}
template<typename _Alloc>
[[__gnu__::__always_inline__]]
constexpr inline void
__alloc_on_move(_Alloc& __one, _Alloc& __two)
{
using __traits = allocator_traits<_Alloc>;
using __pocma
= typename __traits::propagate_on_container_move_assignment::type;
if constexpr (__pocma::value)
__one = std::move(__two);
}
template<typename _Alloc>
[[__gnu__::__always_inline__]]
constexpr inline void
__alloc_on_swap(_Alloc& __one, _Alloc& __two)
{
using __traits = allocator_traits<_Alloc>;
using __pocs = typename __traits::propagate_on_container_swap::type;
if constexpr (__pocs::value)
{
using std::swap;
swap(__one, __two);
}
}
template<typename _Alloc, typename _Tp,
typename _ValueT = __remove_cvref_t<typename _Alloc::value_type>,
typename = void>
struct __is_alloc_insertable_impl
: false_type
{ };
template<typename _Alloc, typename _Tp, typename _ValueT>
struct __is_alloc_insertable_impl<_Alloc, _Tp, _ValueT,
__void_t<decltype(allocator_traits<_Alloc>::construct(
std::declval<_Alloc&>(), std::declval<_ValueT*>(),
std::declval<_Tp>()))>>
: true_type
{ };
template<typename _Alloc>
struct __is_copy_insertable
: __is_alloc_insertable_impl<_Alloc,
typename _Alloc::value_type const&>::type
{ };
template<typename _Tp>
struct __is_copy_insertable<allocator<_Tp>>
: is_copy_constructible<_Tp>
{ };
template<typename _Alloc>
struct __is_move_insertable
: __is_alloc_insertable_impl<_Alloc, typename _Alloc::value_type>::type
{ };
template<typename _Tp>
struct __is_move_insertable<allocator<_Tp>>
: is_move_constructible<_Tp>
{ };
template<typename _Alloc, typename = void>
struct __is_allocator : false_type { };
template<typename _Alloc>
struct __is_allocator<_Alloc,
__void_t<typename _Alloc::value_type,
decltype(std::declval<_Alloc&>().allocate(size_t{}))>>
: true_type { };
template<typename _Alloc>
using _RequireAllocator
= typename enable_if<__is_allocator<_Alloc>::value, _Alloc>::type;
template<typename _Alloc>
using _RequireNotAllocator
= typename enable_if<!__is_allocator<_Alloc>::value, _Alloc>::type;
template<typename _Alloc, bool = __is_empty(_Alloc)>
struct __alloc_swap
{ static void _S_do_it(_Alloc&, _Alloc&) noexcept { } };
template<typename _Alloc>
struct __alloc_swap<_Alloc, false>
{
static void
_S_do_it(_Alloc& __one, _Alloc& __two) noexcept
{
if (__one != __two)
swap(__one, __two);
}
};
template<typename _Tp, bool
= __or_<is_copy_constructible<typename _Tp::value_type>,
is_nothrow_move_constructible<typename _Tp::value_type>>::value>
struct __shrink_to_fit_aux
{ static bool _S_do_it(_Tp&) noexcept { return false; } };
template<typename _Tp>
struct __shrink_to_fit_aux<_Tp, true>
{
static bool
_S_do_it(_Tp& __c) noexcept
{
try
{
_Tp(__make_move_if_noexcept_iterator(__c.begin()),
__make_move_if_noexcept_iterator(__c.end()),
__c.get_allocator()).swap(__c);
return true;
}
catch(...)
{ return false; }
}
};
template<typename _ForwardIterator, typename _Allocator>
void
_Destroy(_ForwardIterator __first, _ForwardIterator __last,
_Allocator& __alloc)
{
for (; __first != __last; ++__first)
allocator_traits<_Allocator>::destroy(__alloc,
std::__addressof(*__first));
}
template<typename _ForwardIterator, typename _Tp>
__attribute__((__always_inline__))
inline void
_Destroy(_ForwardIterator __first, _ForwardIterator __last,
allocator<_Tp>&)
{
std::_Destroy(__first, __last);
}
}
namespace __gnu_cxx __attribute__ ((__visibility__ ("default")))
{
template<typename _Alloc, typename = typename _Alloc::value_type>
struct __alloc_traits
: std::allocator_traits<_Alloc>
{
typedef _Alloc allocator_type;
typedef std::allocator_traits<_Alloc> _Base_type;
typedef typename _Base_type::value_type value_type;
typedef typename _Base_type::pointer pointer;
typedef typename _Base_type::const_pointer const_pointer;
typedef typename _Base_type::size_type size_type;
typedef typename _Base_type::difference_type difference_type;
typedef value_type& reference;
typedef const value_type& const_reference;
using _Base_type::allocate;
using _Base_type::deallocate;
using _Base_type::construct;
using _Base_type::destroy;
using _Base_type::max_size;
private:
template<typename _Ptr>
using __is_custom_pointer
= std::__and_<std::is_same<pointer, _Ptr>,
std::__not_<std::is_pointer<_Ptr>>>;
public:
template<typename _Ptr, typename... _Args>
[[__gnu__::__always_inline__]]
static constexpr
std::__enable_if_t<__is_custom_pointer<_Ptr>::value>
construct(_Alloc& __a, _Ptr __p, _Args&&... __args)
noexcept(noexcept(_Base_type::construct(__a, std::__to_address(__p),
std::forward<_Args>(__args)...)))
{
_Base_type::construct(__a, std::__to_address(__p),
std::forward<_Args>(__args)...);
}
template<typename _Ptr>
[[__gnu__::__always_inline__]]
static constexpr
std::__enable_if_t<__is_custom_pointer<_Ptr>::value>
destroy(_Alloc& __a, _Ptr __p)
noexcept(noexcept(_Base_type::destroy(__a, std::__to_address(__p))))
{ _Base_type::destroy(__a, std::__to_address(__p)); }
[[__gnu__::__always_inline__]]
static constexpr _Alloc _S_select_on_copy(const _Alloc& __a)
{ return _Base_type::select_on_container_copy_construction(__a); }
[[__gnu__::__always_inline__]]
static constexpr void _S_on_swap(_Alloc& __a, _Alloc& __b)
{ std::__alloc_on_swap(__a, __b); }
[[__gnu__::__always_inline__]]
static constexpr bool _S_propagate_on_copy_assign()
{ return _Base_type::propagate_on_container_copy_assignment::value; }
[[__gnu__::__always_inline__]]
static constexpr bool _S_propagate_on_move_assign()
{ return _Base_type::propagate_on_container_move_assignment::value; }
[[__gnu__::__always_inline__]]
static constexpr bool _S_propagate_on_swap()
{ return _Base_type::propagate_on_container_swap::value; }
[[__gnu__::__always_inline__]]
static constexpr bool _S_always_equal()
{ return _Base_type::is_always_equal::value; }
__attribute__((__always_inline__))
static constexpr bool _S_nothrow_move()
{ return _S_propagate_on_move_assign() || _S_always_equal(); }
template<typename _Tp>
struct rebind
{ typedef typename _Base_type::template rebind_alloc<_Tp> other; };
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _ValueType, typename _Tp>
constexpr bool
__check_constructible()
{
static_assert(is_constructible<_ValueType, _Tp>::value,
"result type must be constructible from input type");
return true;
}
template<typename _InputIterator, typename _ForwardIterator>
_ForwardIterator
__do_uninit_copy(_InputIterator __first, _InputIterator __last,
_ForwardIterator __result)
{
_ForwardIterator __cur = __result;
try
{
for (; __first != __last; ++__first, (void)++__cur)
std::_Construct(std::__addressof(*__cur), *__first);
return __cur;
}
catch(...)
{
std::_Destroy(__result, __cur);
throw;
}
}
template<bool _TrivialValueTypes>
struct __uninitialized_copy
{
template<typename _InputIterator, typename _ForwardIterator>
static _ForwardIterator
__uninit_copy(_InputIterator __first, _InputIterator __last,
_ForwardIterator __result)
{ return std::__do_uninit_copy(__first, __last, __result); }
};
template<>
struct __uninitialized_copy<true>
{
template<typename _InputIterator, typename _ForwardIterator>
static _ForwardIterator
__uninit_copy(_InputIterator __first, _InputIterator __last,
_ForwardIterator __result)
{ return std::copy(__first, __last, __result); }
};
template<typename _InputIterator, typename _ForwardIterator>
inline _ForwardIterator
uninitialized_copy(_InputIterator __first, _InputIterator __last,
_ForwardIterator __result)
{
typedef typename iterator_traits<_InputIterator>::value_type
_ValueType1;
typedef typename iterator_traits<_ForwardIterator>::value_type
_ValueType2;
const bool __can_memmove = __is_trivial(_ValueType1);
using _From = decltype(*__first);
const bool __assignable
= __is_trivial(_ValueType2) && __is_assignable(_ValueType2&, _From) && std::__check_constructible<_ValueType2, _From>();
return std::__uninitialized_copy<__can_memmove && __assignable>::
__uninit_copy(__first, __last, __result);
}
template<typename _ForwardIterator, typename _Tp>
void
__do_uninit_fill(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __x)
{
_ForwardIterator __cur = __first;
try
{
for (; __cur != __last; ++__cur)
std::_Construct(std::__addressof(*__cur), __x);
}
catch(...)
{
std::_Destroy(__first, __cur);
throw;
}
}
template<bool _TrivialValueType>
struct __uninitialized_fill
{
template<typename _ForwardIterator, typename _Tp>
static void
__uninit_fill(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __x)
{ std::__do_uninit_fill(__first, __last, __x); }
};
template<>
struct __uninitialized_fill<true>
{
template<typename _ForwardIterator, typename _Tp>
static void
__uninit_fill(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __x)
{ std::fill(__first, __last, __x); }
};
template<typename _ForwardIterator, typename _Tp>
inline void
uninitialized_fill(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __x)
{
typedef typename iterator_traits<_ForwardIterator>::value_type
_ValueType;
const bool __can_fill
= __is_trivial(_ValueType) && __is_assignable(_ValueType&, const _Tp&) && std::__check_constructible<_ValueType, const _Tp&>();
std::__uninitialized_fill<__can_fill>::
__uninit_fill(__first, __last, __x);
}
template<typename _ForwardIterator, typename _Size, typename _Tp>
_ForwardIterator
__do_uninit_fill_n(_ForwardIterator __first, _Size __n, const _Tp& __x)
{
_ForwardIterator __cur = __first;
try
{
for (; __n > 0; --__n, (void) ++__cur)
std::_Construct(std::__addressof(*__cur), __x);
return __cur;
}
catch(...)
{
std::_Destroy(__first, __cur);
throw;
}
}
template<bool _TrivialValueType>
struct __uninitialized_fill_n
{
template<typename _ForwardIterator, typename _Size, typename _Tp>
static _ForwardIterator
__uninit_fill_n(_ForwardIterator __first, _Size __n,
const _Tp& __x)
{ return std::__do_uninit_fill_n(__first, __n, __x); }
};
template<>
struct __uninitialized_fill_n<true>
{
template<typename _ForwardIterator, typename _Size, typename _Tp>
static _ForwardIterator
__uninit_fill_n(_ForwardIterator __first, _Size __n,
const _Tp& __x)
{ return std::fill_n(__first, __n, __x); }
};
template<typename _ForwardIterator, typename _Size, typename _Tp>
inline _ForwardIterator
uninitialized_fill_n(_ForwardIterator __first, _Size __n, const _Tp& __x)
{
typedef typename iterator_traits<_ForwardIterator>::value_type
_ValueType;
const bool __can_fill
= __is_trivial(_ValueType) && __is_assignable(_ValueType&, const _Tp&) && std::__check_constructible<_ValueType, const _Tp&>()
&& __is_integer<_Size>::__value;
return __uninitialized_fill_n<__can_fill>::
__uninit_fill_n(__first, __n, __x);
}
template<typename _InputIterator, typename _ForwardIterator,
typename _Allocator>
_ForwardIterator
__uninitialized_copy_a(_InputIterator __first, _InputIterator __last,
_ForwardIterator __result, _Allocator& __alloc)
{
_ForwardIterator __cur = __result;
try
{
typedef __gnu_cxx::__alloc_traits<_Allocator> __traits;
for (; __first != __last; ++__first, (void)++__cur)
__traits::construct(__alloc, std::__addressof(*__cur), *__first);
return __cur;
}
catch(...)
{
std::_Destroy(__result, __cur, __alloc);
throw;
}
}
template<typename _InputIterator, typename _ForwardIterator, typename _Tp>
inline _ForwardIterator
__uninitialized_copy_a(_InputIterator __first, _InputIterator __last,
_ForwardIterator __result, allocator<_Tp>&)
{
return std::uninitialized_copy(__first, __last, __result);
}
template<typename _InputIterator, typename _ForwardIterator,
typename _Allocator>
inline _ForwardIterator
__uninitialized_move_a(_InputIterator __first, _InputIterator __last,
_ForwardIterator __result, _Allocator& __alloc)
{
return std::__uninitialized_copy_a(std::make_move_iterator(__first),
std::make_move_iterator(__last),
__result, __alloc);
}
template<typename _InputIterator, typename _ForwardIterator,
typename _Allocator>
inline _ForwardIterator
__uninitialized_move_if_noexcept_a(_InputIterator __first,
_InputIterator __last,
_ForwardIterator __result,
_Allocator& __alloc)
{
return std::__uninitialized_copy_a
(std::__make_move_if_noexcept_iterator(__first),
std::__make_move_if_noexcept_iterator(__last), __result, __alloc);
}
template<typename _ForwardIterator, typename _Tp, typename _Allocator>
void
__uninitialized_fill_a(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __x, _Allocator& __alloc)
{
_ForwardIterator __cur = __first;
try
{
typedef __gnu_cxx::__alloc_traits<_Allocator> __traits;
for (; __cur != __last; ++__cur)
__traits::construct(__alloc, std::__addressof(*__cur), __x);
}
catch(...)
{
std::_Destroy(__first, __cur, __alloc);
throw;
}
}
template<typename _ForwardIterator, typename _Tp, typename _Tp2>
inline void
__uninitialized_fill_a(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __x, allocator<_Tp2>&)
{
std::uninitialized_fill(__first, __last, __x);
}
template<typename _ForwardIterator, typename _Size, typename _Tp,
typename _Allocator>
_ForwardIterator
__uninitialized_fill_n_a(_ForwardIterator __first, _Size __n,
const _Tp& __x, _Allocator& __alloc)
{
_ForwardIterator __cur = __first;
try
{
typedef __gnu_cxx::__alloc_traits<_Allocator> __traits;
for (; __n > 0; --__n, (void) ++__cur)
__traits::construct(__alloc, std::__addressof(*__cur), __x);
return __cur;
}
catch(...)
{
std::_Destroy(__first, __cur, __alloc);
throw;
}
}
template<typename _ForwardIterator, typename _Size, typename _Tp,
typename _Tp2>
inline _ForwardIterator
__uninitialized_fill_n_a(_ForwardIterator __first, _Size __n,
const _Tp& __x, allocator<_Tp2>&)
{
return std::uninitialized_fill_n(__first, __n, __x);
}
template<typename _InputIterator1, typename _InputIterator2,
typename _ForwardIterator, typename _Allocator>
inline _ForwardIterator
__uninitialized_copy_move(_InputIterator1 __first1,
_InputIterator1 __last1,
_InputIterator2 __first2,
_InputIterator2 __last2,
_ForwardIterator __result,
_Allocator& __alloc)
{
_ForwardIterator __mid = std::__uninitialized_copy_a(__first1, __last1,
__result,
__alloc);
try
{
return std::__uninitialized_move_a(__first2, __last2, __mid, __alloc);
}
catch(...)
{
std::_Destroy(__result, __mid, __alloc);
throw;
}
}
template<typename _InputIterator1, typename _InputIterator2,
typename _ForwardIterator, typename _Allocator>
inline _ForwardIterator
__uninitialized_move_copy(_InputIterator1 __first1,
_InputIterator1 __last1,
_InputIterator2 __first2,
_InputIterator2 __last2,
_ForwardIterator __result,
_Allocator& __alloc)
{
_ForwardIterator __mid = std::__uninitialized_move_a(__first1, __last1,
__result,
__alloc);
try
{
return std::__uninitialized_copy_a(__first2, __last2, __mid, __alloc);
}
catch(...)
{
std::_Destroy(__result, __mid, __alloc);
throw;
}
}
template<typename _ForwardIterator, typename _Tp, typename _InputIterator,
typename _Allocator>
inline _ForwardIterator
__uninitialized_fill_move(_ForwardIterator __result, _ForwardIterator __mid,
const _Tp& __x, _InputIterator __first,
_InputIterator __last, _Allocator& __alloc)
{
std::__uninitialized_fill_a(__result, __mid, __x, __alloc);
try
{
return std::__uninitialized_move_a(__first, __last, __mid, __alloc);
}
catch(...)
{
std::_Destroy(__result, __mid, __alloc);
throw;
}
}
template<typename _InputIterator, typename _ForwardIterator, typename _Tp,
typename _Allocator>
inline void
__uninitialized_move_fill(_InputIterator __first1, _InputIterator __last1,
_ForwardIterator __first2,
_ForwardIterator __last2, const _Tp& __x,
_Allocator& __alloc)
{
_ForwardIterator __mid2 = std::__uninitialized_move_a(__first1, __last1,
__first2,
__alloc);
try
{
std::__uninitialized_fill_a(__mid2, __last2, __x, __alloc);
}
catch(...)
{
std::_Destroy(__first2, __mid2, __alloc);
throw;
}
}
template<bool _TrivialValueType>
struct __uninitialized_default_1
{
template<typename _ForwardIterator>
static void
__uninit_default(_ForwardIterator __first, _ForwardIterator __last)
{
_ForwardIterator __cur = __first;
try
{
for (; __cur != __last; ++__cur)
std::_Construct(std::__addressof(*__cur));
}
catch(...)
{
std::_Destroy(__first, __cur);
throw;
}
}
};
template<>
struct __uninitialized_default_1<true>
{
template<typename _ForwardIterator>
static void
__uninit_default(_ForwardIterator __first, _ForwardIterator __last)
{
if (__first == __last)
return;
typename iterator_traits<_ForwardIterator>::value_type* __val
= std::__addressof(*__first);
std::_Construct(__val);
if (++__first != __last)
std::fill(__first, __last, *__val);
}
};
template<bool _TrivialValueType>
struct __uninitialized_default_n_1
{
template<typename _ForwardIterator, typename _Size>
static _ForwardIterator
__uninit_default_n(_ForwardIterator __first, _Size __n)
{
_ForwardIterator __cur = __first;
try
{
for (; __n > 0; --__n, (void) ++__cur)
std::_Construct(std::__addressof(*__cur));
return __cur;
}
catch(...)
{
std::_Destroy(__first, __cur);
throw;
}
}
};
template<>
struct __uninitialized_default_n_1<true>
{
template<typename _ForwardIterator, typename _Size>
static _ForwardIterator
__uninit_default_n(_ForwardIterator __first, _Size __n)
{
if (__n > 0)
{
typename iterator_traits<_ForwardIterator>::value_type* __val
= std::__addressof(*__first);
std::_Construct(__val);
++__first;
__first = std::fill_n(__first, __n - 1, *__val);
}
return __first;
}
};
template<typename _ForwardIterator>
inline void
__uninitialized_default(_ForwardIterator __first,
_ForwardIterator __last)
{
typedef typename iterator_traits<_ForwardIterator>::value_type
_ValueType;
const bool __assignable = is_copy_assignable<_ValueType>::value;
std::__uninitialized_default_1<__is_trivial(_ValueType)
&& __assignable>::
__uninit_default(__first, __last);
}
template<typename _ForwardIterator, typename _Size>
inline _ForwardIterator
__uninitialized_default_n(_ForwardIterator __first, _Size __n)
{
typedef typename iterator_traits<_ForwardIterator>::value_type
_ValueType;
constexpr bool __can_fill
= __and_<is_integral<_Size>, is_copy_assignable<_ValueType>>::value;
return __uninitialized_default_n_1<__is_trivial(_ValueType)
&& __can_fill>::
__uninit_default_n(__first, __n);
}
template<typename _ForwardIterator, typename _Allocator>
void
__uninitialized_default_a(_ForwardIterator __first,
_ForwardIterator __last,
_Allocator& __alloc)
{
_ForwardIterator __cur = __first;
try
{
typedef __gnu_cxx::__alloc_traits<_Allocator> __traits;
for (; __cur != __last; ++__cur)
__traits::construct(__alloc, std::__addressof(*__cur));
}
catch(...)
{
std::_Destroy(__first, __cur, __alloc);
throw;
}
}
template<typename _ForwardIterator, typename _Tp>
inline void
__uninitialized_default_a(_ForwardIterator __first,
_ForwardIterator __last,
allocator<_Tp>&)
{ std::__uninitialized_default(__first, __last); }
template<typename _ForwardIterator, typename _Size, typename _Allocator>
_ForwardIterator
__uninitialized_default_n_a(_ForwardIterator __first, _Size __n,
_Allocator& __alloc)
{
_ForwardIterator __cur = __first;
try
{
typedef __gnu_cxx::__alloc_traits<_Allocator> __traits;
for (; __n > 0; --__n, (void) ++__cur)
__traits::construct(__alloc, std::__addressof(*__cur));
return __cur;
}
catch(...)
{
std::_Destroy(__first, __cur, __alloc);
throw;
}
}
template<typename _ForwardIterator, typename _Size, typename _Tp>
inline _ForwardIterator
__uninitialized_default_n_a(_ForwardIterator __first, _Size __n,
allocator<_Tp>&)
{ return std::__uninitialized_default_n(__first, __n); }
template<bool _TrivialValueType>
struct __uninitialized_default_novalue_1
{
template<typename _ForwardIterator>
static void
__uninit_default_novalue(_ForwardIterator __first,
_ForwardIterator __last)
{
_ForwardIterator __cur = __first;
try
{
for (; __cur != __last; ++__cur)
std::_Construct_novalue(std::__addressof(*__cur));
}
catch(...)
{
std::_Destroy(__first, __cur);
throw;
}
}
};
template<>
struct __uninitialized_default_novalue_1<true>
{
template<typename _ForwardIterator>
static void
__uninit_default_novalue(_ForwardIterator __first,
_ForwardIterator __last)
{
}
};
template<bool _TrivialValueType>
struct __uninitialized_default_novalue_n_1
{
template<typename _ForwardIterator, typename _Size>
static _ForwardIterator
__uninit_default_novalue_n(_ForwardIterator __first, _Size __n)
{
_ForwardIterator __cur = __first;
try
{
for (; __n > 0; --__n, (void) ++__cur)
std::_Construct_novalue(std::__addressof(*__cur));
return __cur;
}
catch(...)
{
std::_Destroy(__first, __cur);
throw;
}
}
};
template<>
struct __uninitialized_default_novalue_n_1<true>
{
template<typename _ForwardIterator, typename _Size>
static _ForwardIterator
__uninit_default_novalue_n(_ForwardIterator __first, _Size __n)
{ return std::next(__first, __n); }
};
template<typename _ForwardIterator>
inline void
__uninitialized_default_novalue(_ForwardIterator __first,
_ForwardIterator __last)
{
typedef typename iterator_traits<_ForwardIterator>::value_type
_ValueType;
std::__uninitialized_default_novalue_1<
is_trivially_default_constructible<_ValueType>::value>::
__uninit_default_novalue(__first, __last);
}
template<typename _ForwardIterator, typename _Size>
inline _ForwardIterator
__uninitialized_default_novalue_n(_ForwardIterator __first, _Size __n)
{
typedef typename iterator_traits<_ForwardIterator>::value_type
_ValueType;
return __uninitialized_default_novalue_n_1<
is_trivially_default_constructible<_ValueType>::value>::
__uninit_default_novalue_n(__first, __n);
}
template<typename _InputIterator, typename _Size,
typename _ForwardIterator>
_ForwardIterator
__uninitialized_copy_n(_InputIterator __first, _Size __n,
_ForwardIterator __result, input_iterator_tag)
{
_ForwardIterator __cur = __result;
try
{
for (; __n > 0; --__n, (void) ++__first, ++__cur)
std::_Construct(std::__addressof(*__cur), *__first);
return __cur;
}
catch(...)
{
std::_Destroy(__result, __cur);
throw;
}
}
template<typename _RandomAccessIterator, typename _Size,
typename _ForwardIterator>
inline _ForwardIterator
__uninitialized_copy_n(_RandomAccessIterator __first, _Size __n,
_ForwardIterator __result,
random_access_iterator_tag)
{ return std::uninitialized_copy(__first, __first + __n, __result); }
template<typename _InputIterator, typename _Size,
typename _ForwardIterator>
pair<_InputIterator, _ForwardIterator>
__uninitialized_copy_n_pair(_InputIterator __first, _Size __n,
_ForwardIterator __result, input_iterator_tag)
{
_ForwardIterator __cur = __result;
try
{
for (; __n > 0; --__n, (void) ++__first, ++__cur)
std::_Construct(std::__addressof(*__cur), *__first);
return {__first, __cur};
}
catch(...)
{
std::_Destroy(__result, __cur);
throw;
}
}
template<typename _RandomAccessIterator, typename _Size,
typename _ForwardIterator>
inline pair<_RandomAccessIterator, _ForwardIterator>
__uninitialized_copy_n_pair(_RandomAccessIterator __first, _Size __n,
_ForwardIterator __result,
random_access_iterator_tag)
{
auto __second_res = uninitialized_copy(__first, __first + __n, __result);
auto __first_res = std::next(__first, __n);
return {__first_res, __second_res};
}
template<typename _InputIterator, typename _Size, typename _ForwardIterator>
inline _ForwardIterator
uninitialized_copy_n(_InputIterator __first, _Size __n,
_ForwardIterator __result)
{ return std::__uninitialized_copy_n(__first, __n, __result,
std::__iterator_category(__first)); }
template<typename _InputIterator, typename _Size, typename _ForwardIterator>
inline pair<_InputIterator, _ForwardIterator>
__uninitialized_copy_n_pair(_InputIterator __first, _Size __n,
_ForwardIterator __result)
{
return
std::__uninitialized_copy_n_pair(__first, __n, __result,
std::__iterator_category(__first));
}
template <typename _ForwardIterator>
inline void
uninitialized_default_construct(_ForwardIterator __first,
_ForwardIterator __last)
{
__uninitialized_default_novalue(__first, __last);
}
template <typename _ForwardIterator, typename _Size>
inline _ForwardIterator
uninitialized_default_construct_n(_ForwardIterator __first, _Size __count)
{
return __uninitialized_default_novalue_n(__first, __count);
}
template <typename _ForwardIterator>
inline void
uninitialized_value_construct(_ForwardIterator __first,
_ForwardIterator __last)
{
return __uninitialized_default(__first, __last);
}
template <typename _ForwardIterator, typename _Size>
inline _ForwardIterator
uninitialized_value_construct_n(_ForwardIterator __first, _Size __count)
{
return __uninitialized_default_n(__first, __count);
}
template <typename _InputIterator, typename _ForwardIterator>
inline _ForwardIterator
uninitialized_move(_InputIterator __first, _InputIterator __last,
_ForwardIterator __result)
{
return std::uninitialized_copy
(std::make_move_iterator(__first),
std::make_move_iterator(__last), __result);
}
template <typename _InputIterator, typename _Size, typename _ForwardIterator>
inline pair<_InputIterator, _ForwardIterator>
uninitialized_move_n(_InputIterator __first, _Size __count,
_ForwardIterator __result)
{
auto __res = std::__uninitialized_copy_n_pair
(std::make_move_iterator(__first),
__count, __result);
return {__res.first.base(), __res.second};
}
template<typename _Tp, typename _Up, typename _Allocator>
inline void
__relocate_object_a(_Tp* __restrict __dest, _Up* __restrict __orig,
_Allocator& __alloc)
noexcept(noexcept(std::allocator_traits<_Allocator>::construct(__alloc,
__dest, std::move(*__orig)))
&& noexcept(std::allocator_traits<_Allocator>::destroy(
__alloc, std::__addressof(*__orig))))
{
typedef std::allocator_traits<_Allocator> __traits;
__traits::construct(__alloc, __dest, std::move(*__orig));
__traits::destroy(__alloc, std::__addressof(*__orig));
}
template<typename _Tp, typename = void>
struct __is_bitwise_relocatable
: is_trivial<_Tp> { };
template <typename _InputIterator, typename _ForwardIterator,
typename _Allocator>
inline _ForwardIterator
__relocate_a_1(_InputIterator __first, _InputIterator __last,
_ForwardIterator __result, _Allocator& __alloc)
noexcept(noexcept(std::__relocate_object_a(std::addressof(*__result),
std::addressof(*__first),
__alloc)))
{
typedef typename iterator_traits<_InputIterator>::value_type
_ValueType;
typedef typename iterator_traits<_ForwardIterator>::value_type
_ValueType2;
static_assert(std::is_same<_ValueType, _ValueType2>::value,
"relocation is only possible for values of the same type");
_ForwardIterator __cur = __result;
for (; __first != __last; ++__first, (void)++__cur)
std::__relocate_object_a(std::__addressof(*__cur),
std::__addressof(*__first), __alloc);
return __cur;
}
template <typename _Tp, typename _Up>
inline __enable_if_t<std::__is_bitwise_relocatable<_Tp>::value, _Tp*>
__relocate_a_1(_Tp* __first, _Tp* __last,
_Tp* __result,
[[__maybe_unused__]] allocator<_Up>& __alloc) noexcept
{
ptrdiff_t __count = __last - __first;
if (__count > 0)
{
__builtin_memmove(__result, __first, __count * sizeof(_Tp));
}
return __result + __count;
}
template <typename _InputIterator, typename _ForwardIterator,
typename _Allocator>
inline _ForwardIterator
__relocate_a(_InputIterator __first, _InputIterator __last,
_ForwardIterator __result, _Allocator& __alloc)
noexcept(noexcept(__relocate_a_1(std::__niter_base(__first),
std::__niter_base(__last),
std::__niter_base(__result), __alloc)))
{
return std::__relocate_a_1(std::__niter_base(__first),
std::__niter_base(__last),
std::__niter_base(__result), __alloc);
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<class _E>
class initializer_list
{
public:
typedef _E value_type;
typedef const _E& reference;
typedef const _E& const_reference;
typedef size_t size_type;
typedef const _E* iterator;
typedef const _E* const_iterator;
private:
iterator _M_array;
size_type _M_len;
constexpr initializer_list(const_iterator __a, size_type __l)
: _M_array(__a), _M_len(__l) { }
public:
constexpr initializer_list() noexcept
: _M_array(0), _M_len(0) { }
constexpr size_type
size() const noexcept { return _M_len; }
constexpr const_iterator
begin() const noexcept { return _M_array; }
constexpr const_iterator
end() const noexcept { return begin() + size(); }
};
template<class _Tp>
constexpr const _Tp*
begin(initializer_list<_Tp> __ils) noexcept
{ return __ils.begin(); }
template<class _Tp>
constexpr const _Tp*
end(initializer_list<_Tp> __ils) noexcept
{ return __ils.end(); }
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Tp, typename _Alloc>
struct _Vector_base
{
typedef typename __gnu_cxx::__alloc_traits<_Alloc>::template
rebind<_Tp>::other _Tp_alloc_type;
typedef typename __gnu_cxx::__alloc_traits<_Tp_alloc_type>::pointer
pointer;
struct _Vector_impl_data
{
pointer _M_start;
pointer _M_finish;
pointer _M_end_of_storage;
_Vector_impl_data() noexcept
: _M_start(), _M_finish(), _M_end_of_storage()
{ }
_Vector_impl_data(_Vector_impl_data&& __x) noexcept
: _M_start(__x._M_start), _M_finish(__x._M_finish),
_M_end_of_storage(__x._M_end_of_storage)
{ __x._M_start = __x._M_finish = __x._M_end_of_storage = pointer(); }
void
_M_copy_data(_Vector_impl_data const& __x) noexcept
{
_M_start = __x._M_start;
_M_finish = __x._M_finish;
_M_end_of_storage = __x._M_end_of_storage;
}
void
_M_swap_data(_Vector_impl_data& __x) noexcept
{
_Vector_impl_data __tmp;
__tmp._M_copy_data(*this);
_M_copy_data(__x);
__x._M_copy_data(__tmp);
}
};
struct _Vector_impl
: public _Tp_alloc_type, public _Vector_impl_data
{
_Vector_impl() noexcept(is_nothrow_default_constructible<_Tp_alloc_type>::value)
: _Tp_alloc_type()
{ }
_Vector_impl(_Tp_alloc_type const& __a) noexcept
: _Tp_alloc_type(__a)
{ }
_Vector_impl(_Vector_impl&& __x) noexcept
: _Tp_alloc_type(std::move(__x)), _Vector_impl_data(std::move(__x))
{ }
_Vector_impl(_Tp_alloc_type&& __a) noexcept
: _Tp_alloc_type(std::move(__a))
{ }
_Vector_impl(_Tp_alloc_type&& __a, _Vector_impl&& __rv) noexcept
: _Tp_alloc_type(std::move(__a)), _Vector_impl_data(std::move(__rv))
{ }
};
public:
typedef _Alloc allocator_type;
_Tp_alloc_type&
_M_get_Tp_allocator() noexcept
{ return this->_M_impl; }
const _Tp_alloc_type&
_M_get_Tp_allocator() const noexcept
{ return this->_M_impl; }
allocator_type
get_allocator() const noexcept
{ return allocator_type(_M_get_Tp_allocator()); }
_Vector_base() = default;
_Vector_base(const allocator_type& __a) noexcept
: _M_impl(__a) { }
_Vector_base(size_t __n)
: _M_impl()
{ _M_create_storage(__n); }
_Vector_base(size_t __n, const allocator_type& __a)
: _M_impl(__a)
{ _M_create_storage(__n); }
_Vector_base(_Vector_base&&) = default;
_Vector_base(_Tp_alloc_type&& __a) noexcept
: _M_impl(std::move(__a)) { }
_Vector_base(_Vector_base&& __x, const allocator_type& __a)
: _M_impl(__a)
{
if (__x.get_allocator() == __a)
this->_M_impl._M_swap_data(__x._M_impl);
else
{
size_t __n = __x._M_impl._M_finish - __x._M_impl._M_start;
_M_create_storage(__n);
}
}
_Vector_base(const allocator_type& __a, _Vector_base&& __x)
: _M_impl(_Tp_alloc_type(__a), std::move(__x._M_impl))
{ }
~_Vector_base() noexcept
{
_M_deallocate(_M_impl._M_start,
_M_impl._M_end_of_storage - _M_impl._M_start);
}
public:
_Vector_impl _M_impl;
pointer
_M_allocate(size_t __n)
{
typedef __gnu_cxx::__alloc_traits<_Tp_alloc_type> _Tr;
return __n != 0 ? _Tr::allocate(_M_impl, __n) : pointer();
}
void
_M_deallocate(pointer __p, size_t __n)
{
typedef __gnu_cxx::__alloc_traits<_Tp_alloc_type> _Tr;
if (__p)
_Tr::deallocate(_M_impl, __p, __n);
}
protected:
void
_M_create_storage(size_t __n)
{
this->_M_impl._M_start = this->_M_allocate(__n);
this->_M_impl._M_finish = this->_M_impl._M_start;
this->_M_impl._M_end_of_storage = this->_M_impl._M_start + __n;
}
};
template<typename _Tp, typename _Alloc = std::allocator<_Tp> >
class vector : protected _Vector_base<_Tp, _Alloc>
{
static_assert(is_same<typename remove_cv<_Tp>::type, _Tp>::value,
"std::vector must have a non-const, non-volatile value_type");
typedef _Vector_base<_Tp, _Alloc> _Base;
typedef typename _Base::_Tp_alloc_type _Tp_alloc_type;
typedef __gnu_cxx::__alloc_traits<_Tp_alloc_type> _Alloc_traits;
public:
typedef _Tp value_type;
typedef typename _Base::pointer pointer;
typedef typename _Alloc_traits::const_pointer const_pointer;
typedef typename _Alloc_traits::reference reference;
typedef typename _Alloc_traits::const_reference const_reference;
typedef __gnu_cxx::__normal_iterator<pointer, vector> iterator;
typedef __gnu_cxx::__normal_iterator<const_pointer, vector>
const_iterator;
typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
typedef std::reverse_iterator<iterator> reverse_iterator;
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef _Alloc allocator_type;
private:
static constexpr bool
_S_nothrow_relocate(true_type)
{
return noexcept(std::__relocate_a(std::declval<pointer>(),
std::declval<pointer>(),
std::declval<pointer>(),
std::declval<_Tp_alloc_type&>()));
}
static constexpr bool
_S_nothrow_relocate(false_type)
{ return false; }
static constexpr bool
_S_use_relocate()
{
return _S_nothrow_relocate(__is_move_insertable<_Tp_alloc_type>{});
}
static pointer
_S_do_relocate(pointer __first, pointer __last, pointer __result,
_Tp_alloc_type& __alloc, true_type) noexcept
{
return std::__relocate_a(__first, __last, __result, __alloc);
}
static pointer
_S_do_relocate(pointer, pointer, pointer __result,
_Tp_alloc_type&, false_type) noexcept
{ return __result; }
static pointer
_S_relocate(pointer __first, pointer __last, pointer __result,
_Tp_alloc_type& __alloc) noexcept
{
return std::__relocate_a(__first, __last, __result, __alloc);
}
protected:
using _Base::_M_allocate;
using _Base::_M_deallocate;
using _Base::_M_impl;
using _Base::_M_get_Tp_allocator;
public:
vector() = default;
explicit
vector(const allocator_type& __a) noexcept
: _Base(__a) { }
explicit
vector(size_type __n, const allocator_type& __a = allocator_type())
: _Base(_S_check_init_len(__n, __a), __a)
{ _M_default_initialize(__n); }
vector(size_type __n, const value_type& __value,
const allocator_type& __a = allocator_type())
: _Base(_S_check_init_len(__n, __a), __a)
{ _M_fill_initialize(__n, __value); }
vector(const vector& __x)
: _Base(__x.size(),
_Alloc_traits::_S_select_on_copy(__x._M_get_Tp_allocator()))
{
this->_M_impl._M_finish =
std::__uninitialized_copy_a(__x.begin(), __x.end(),
this->_M_impl._M_start,
_M_get_Tp_allocator());
}
vector(vector&&) noexcept = default;
vector(const vector& __x, const __type_identity_t<allocator_type>& __a)
: _Base(__x.size(), __a)
{
this->_M_impl._M_finish =
std::__uninitialized_copy_a(__x.begin(), __x.end(),
this->_M_impl._M_start,
_M_get_Tp_allocator());
}
private:
vector(vector&& __rv, const allocator_type& __m, true_type) noexcept
: _Base(__m, std::move(__rv))
{ }
vector(vector&& __rv, const allocator_type& __m, false_type)
: _Base(__m)
{
if (__rv.get_allocator() == __m)
this->_M_impl._M_swap_data(__rv._M_impl);
else if (!__rv.empty())
{
this->_M_create_storage(__rv.size());
this->_M_impl._M_finish =
std::__uninitialized_move_a(__rv.begin(), __rv.end(),
this->_M_impl._M_start,
_M_get_Tp_allocator());
__rv.clear();
}
}
public:
vector(vector&& __rv, const __type_identity_t<allocator_type>& __m)
noexcept( noexcept(
vector(std::declval<vector&&>(), std::declval<const allocator_type&>(),
std::declval<typename _Alloc_traits::is_always_equal>())) )
: vector(std::move(__rv), __m, typename _Alloc_traits::is_always_equal{})
{ }
vector(initializer_list<value_type> __l,
const allocator_type& __a = allocator_type())
: _Base(__a)
{
_M_range_initialize(__l.begin(), __l.end(),
random_access_iterator_tag());
}
template<typename _InputIterator,
typename = std::_RequireInputIter<_InputIterator>>
vector(_InputIterator __first, _InputIterator __last,
const allocator_type& __a = allocator_type())
: _Base(__a)
{
_M_range_initialize(__first, __last,
std::__iterator_category(__first));
}
~vector() noexcept
{
std::_Destroy(this->_M_impl._M_start, this->_M_impl._M_finish,
_M_get_Tp_allocator());
;
}
vector&
operator=(const vector& __x);
vector&
operator=(vector&& __x) noexcept(_Alloc_traits::_S_nothrow_move())
{
constexpr bool __move_storage =
_Alloc_traits::_S_propagate_on_move_assign()
|| _Alloc_traits::_S_always_equal();
_M_move_assign(std::move(__x), __bool_constant<__move_storage>());
return *this;
}
vector&
operator=(initializer_list<value_type> __l)
{
this->_M_assign_aux(__l.begin(), __l.end(),
random_access_iterator_tag());
return *this;
}
void
assign(size_type __n, const value_type& __val)
{ _M_fill_assign(__n, __val); }
template<typename _InputIterator,
typename = std::_RequireInputIter<_InputIterator>>
void
assign(_InputIterator __first, _InputIterator __last)
{ _M_assign_aux(__first, __last, std::__iterator_category(__first)); }
void
assign(initializer_list<value_type> __l)
{
this->_M_assign_aux(__l.begin(), __l.end(),
random_access_iterator_tag());
}
using _Base::get_allocator;
[[__nodiscard__]]
iterator
begin() noexcept
{ return iterator(this->_M_impl._M_start); }
[[__nodiscard__]]
const_iterator
begin() const noexcept
{ return const_iterator(this->_M_impl._M_start); }
[[__nodiscard__]]
iterator
end() noexcept
{ return iterator(this->_M_impl._M_finish); }
[[__nodiscard__]]
const_iterator
end() const noexcept
{ return const_iterator(this->_M_impl._M_finish); }
[[__nodiscard__]]
reverse_iterator
rbegin() noexcept
{ return reverse_iterator(end()); }
[[__nodiscard__]]
const_reverse_iterator
rbegin() const noexcept
{ return const_reverse_iterator(end()); }
[[__nodiscard__]]
reverse_iterator
rend() noexcept
{ return reverse_iterator(begin()); }
[[__nodiscard__]]
const_reverse_iterator
rend() const noexcept
{ return const_reverse_iterator(begin()); }
[[__nodiscard__]]
const_iterator
cbegin() const noexcept
{ return const_iterator(this->_M_impl._M_start); }
[[__nodiscard__]]
const_iterator
cend() const noexcept
{ return const_iterator(this->_M_impl._M_finish); }
[[__nodiscard__]]
const_reverse_iterator
crbegin() const noexcept
{ return const_reverse_iterator(end()); }
[[__nodiscard__]]
const_reverse_iterator
crend() const noexcept
{ return const_reverse_iterator(begin()); }
[[__nodiscard__]]
size_type
size() const noexcept
{ return size_type(this->_M_impl._M_finish - this->_M_impl._M_start); }
[[__nodiscard__]]
size_type
max_size() const noexcept
{ return _S_max_size(_M_get_Tp_allocator()); }
void
resize(size_type __new_size)
{
if (__new_size > size())
_M_default_append(__new_size - size());
else if (__new_size < size())
_M_erase_at_end(this->_M_impl._M_start + __new_size);
}
void
resize(size_type __new_size, const value_type& __x)
{
if (__new_size > size())
_M_fill_insert(end(), __new_size - size(), __x);
else if (__new_size < size())
_M_erase_at_end(this->_M_impl._M_start + __new_size);
}
void
shrink_to_fit()
{ _M_shrink_to_fit(); }
[[__nodiscard__]]
size_type
capacity() const noexcept
{ return size_type(this->_M_impl._M_end_of_storage
- this->_M_impl._M_start); }
[[__nodiscard__]]
bool
empty() const noexcept
{ return begin() == end(); }
void
reserve(size_type __n);
[[__nodiscard__]]
reference
operator[](size_type __n) noexcept
{
;
return *(this->_M_impl._M_start + __n);
}
[[__nodiscard__]]
const_reference
operator[](size_type __n) const noexcept
{
;
return *(this->_M_impl._M_start + __n);
}
protected:
void
_M_range_check(size_type __n) const
{
if (__n >= this->size())
__throw_out_of_range_fmt(("vector::_M_range_check: __n " "(which is %zu) >= this->size() " "(which is %zu)")
,
__n, this->size());
}
public:
reference
at(size_type __n)
{
_M_range_check(__n);
return (*this)[__n];
}
const_reference
at(size_type __n) const
{
_M_range_check(__n);
return (*this)[__n];
}
[[__nodiscard__]]
reference
front() noexcept
{
;
return *begin();
}
[[__nodiscard__]]
const_reference
front() const noexcept
{
;
return *begin();
}
[[__nodiscard__]]
reference
back() noexcept
{
;
return *(end() - 1);
}
[[__nodiscard__]]
const_reference
back() const noexcept
{
;
return *(end() - 1);
}
[[__nodiscard__]]
_Tp*
data() noexcept
{ return _M_data_ptr(this->_M_impl._M_start); }
[[__nodiscard__]]
const _Tp*
data() const noexcept
{ return _M_data_ptr(this->_M_impl._M_start); }
void
push_back(const value_type& __x)
{
if (this->_M_impl._M_finish != this->_M_impl._M_end_of_storage)
{
;
_Alloc_traits::construct(this->_M_impl, this->_M_impl._M_finish,
__x);
++this->_M_impl._M_finish;
;
}
else
_M_realloc_insert(end(), __x);
}
void
push_back(value_type&& __x)
{ emplace_back(std::move(__x)); }
template<typename... _Args>
reference
emplace_back(_Args&&... __args);
void
pop_back() noexcept
{
;
--this->_M_impl._M_finish;
_Alloc_traits::destroy(this->_M_impl, this->_M_impl._M_finish);
;
}
template<typename... _Args>
iterator
emplace(const_iterator __position, _Args&&... __args)
{ return _M_emplace_aux(__position, std::forward<_Args>(__args)...); }
iterator
insert(const_iterator __position, const value_type& __x);
iterator
insert(const_iterator __position, value_type&& __x)
{ return _M_insert_rval(__position, std::move(__x)); }
iterator
insert(const_iterator __position, initializer_list<value_type> __l)
{
auto __offset = __position - cbegin();
_M_range_insert(begin() + __offset, __l.begin(), __l.end(),
std::random_access_iterator_tag());
return begin() + __offset;
}
iterator
insert(const_iterator __position, size_type __n, const value_type& __x)
{
difference_type __offset = __position - cbegin();
_M_fill_insert(begin() + __offset, __n, __x);
return begin() + __offset;
}
template<typename _InputIterator,
typename = std::_RequireInputIter<_InputIterator>>
iterator
insert(const_iterator __position, _InputIterator __first,
_InputIterator __last)
{
difference_type __offset = __position - cbegin();
_M_range_insert(begin() + __offset, __first, __last,
std::__iterator_category(__first));
return begin() + __offset;
}
iterator
erase(const_iterator __position)
{ return _M_erase(begin() + (__position - cbegin())); }
iterator
erase(const_iterator __first, const_iterator __last)
{
const auto __beg = begin();
const auto __cbeg = cbegin();
return _M_erase(__beg + (__first - __cbeg), __beg + (__last - __cbeg));
}
void
swap(vector& __x) noexcept
{
do { if (std::__is_constant_evaluated() && !bool(_Alloc_traits::propagate_on_container_swap::value || _M_get_Tp_allocator() == __x._M_get_Tp_allocator())) __builtin_unreachable(); } while (false)
;
this->_M_impl._M_swap_data(__x._M_impl);
_Alloc_traits::_S_on_swap(_M_get_Tp_allocator(),
__x._M_get_Tp_allocator());
}
void
clear() noexcept
{ _M_erase_at_end(this->_M_impl._M_start); }
protected:
template<typename _ForwardIterator>
pointer
_M_allocate_and_copy(size_type __n,
_ForwardIterator __first, _ForwardIterator __last)
{
pointer __result = this->_M_allocate(__n);
try
{
std::__uninitialized_copy_a(__first, __last, __result,
_M_get_Tp_allocator());
return __result;
}
catch(...)
{
_M_deallocate(__result, __n);
throw;
}
}
template<typename _InputIterator>
void
_M_range_initialize(_InputIterator __first, _InputIterator __last,
std::input_iterator_tag)
{
try {
for (; __first != __last; ++__first)
emplace_back(*__first);
} catch(...) {
clear();
throw;
}
}
template<typename _ForwardIterator>
void
_M_range_initialize(_ForwardIterator __first, _ForwardIterator __last,
std::forward_iterator_tag)
{
const size_type __n = std::distance(__first, __last);
this->_M_impl._M_start
= this->_M_allocate(_S_check_init_len(__n, _M_get_Tp_allocator()));
this->_M_impl._M_end_of_storage = this->_M_impl._M_start + __n;
this->_M_impl._M_finish =
std::__uninitialized_copy_a(__first, __last,
this->_M_impl._M_start,
_M_get_Tp_allocator());
}
void
_M_fill_initialize(size_type __n, const value_type& __value)
{
this->_M_impl._M_finish =
std::__uninitialized_fill_n_a(this->_M_impl._M_start, __n, __value,
_M_get_Tp_allocator());
}
void
_M_default_initialize(size_type __n)
{
this->_M_impl._M_finish =
std::__uninitialized_default_n_a(this->_M_impl._M_start, __n,
_M_get_Tp_allocator());
}
template<typename _Integer>
void
_M_assign_dispatch(_Integer __n, _Integer __val, __true_type)
{ _M_fill_assign(__n, __val); }
template<typename _InputIterator>
void
_M_assign_dispatch(_InputIterator __first, _InputIterator __last,
__false_type)
{ _M_assign_aux(__first, __last, std::__iterator_category(__first)); }
template<typename _InputIterator>
void
_M_assign_aux(_InputIterator __first, _InputIterator __last,
std::input_iterator_tag);
template<typename _ForwardIterator>
void
_M_assign_aux(_ForwardIterator __first, _ForwardIterator __last,
std::forward_iterator_tag);
void
_M_fill_assign(size_type __n, const value_type& __val);
template<typename _Integer>
void
_M_insert_dispatch(iterator __pos, _Integer __n, _Integer __val,
__true_type)
{ _M_fill_insert(__pos, __n, __val); }
template<typename _InputIterator>
void
_M_insert_dispatch(iterator __pos, _InputIterator __first,
_InputIterator __last, __false_type)
{
_M_range_insert(__pos, __first, __last,
std::__iterator_category(__first));
}
template<typename _InputIterator>
void
_M_range_insert(iterator __pos, _InputIterator __first,
_InputIterator __last, std::input_iterator_tag);
template<typename _ForwardIterator>
void
_M_range_insert(iterator __pos, _ForwardIterator __first,
_ForwardIterator __last, std::forward_iterator_tag);
void
_M_fill_insert(iterator __pos, size_type __n, const value_type& __x);
void
_M_default_append(size_type __n);
bool
_M_shrink_to_fit();
struct _Temporary_value
{
template<typename... _Args>
explicit
_Temporary_value(vector* __vec, _Args&&... __args) : _M_this(__vec)
{
_Alloc_traits::construct(_M_this->_M_impl, _M_ptr(),
std::forward<_Args>(__args)...);
}
~_Temporary_value()
{ _Alloc_traits::destroy(_M_this->_M_impl, _M_ptr()); }
value_type&
_M_val() noexcept { return _M_storage._M_val; }
private:
_Tp*
_M_ptr() noexcept { return std::__addressof(_M_storage._M_val); }
union _Storage
{
constexpr _Storage() : _M_byte() { }
~_Storage() { }
_Storage& operator=(const _Storage&) = delete;
unsigned char _M_byte;
_Tp _M_val;
};
vector* _M_this;
_Storage _M_storage;
};
template<typename _Arg>
void
_M_insert_aux(iterator __position, _Arg&& __arg);
template<typename... _Args>
void
_M_realloc_insert(iterator __position, _Args&&... __args);
iterator
_M_insert_rval(const_iterator __position, value_type&& __v);
template<typename... _Args>
iterator
_M_emplace_aux(const_iterator __position, _Args&&... __args);
iterator
_M_emplace_aux(const_iterator __position, value_type&& __v)
{ return _M_insert_rval(__position, std::move(__v)); }
size_type
_M_check_len(size_type __n, const char* __s) const
{
if (max_size() - size() < __n)
__throw_length_error((__s));
const size_type __len = size() + (std::max)(size(), __n);
return (__len < size() || __len > max_size()) ? max_size() : __len;
}
static size_type
_S_check_init_len(size_type __n, const allocator_type& __a)
{
if (__n > _S_max_size(_Tp_alloc_type(__a)))
__throw_length_error(
("cannot create std::vector larger than max_size()"));
return __n;
}
static size_type
_S_max_size(const _Tp_alloc_type& __a) noexcept
{
const size_t __diffmax
= __gnu_cxx::__numeric_traits<ptrdiff_t>::__max / sizeof(_Tp);
const size_t __allocmax = _Alloc_traits::max_size(__a);
return (std::min)(__diffmax, __allocmax);
}
void
_M_erase_at_end(pointer __pos) noexcept
{
if (size_type __n = this->_M_impl._M_finish - __pos)
{
std::_Destroy(__pos, this->_M_impl._M_finish,
_M_get_Tp_allocator());
this->_M_impl._M_finish = __pos;
;
}
}
iterator
_M_erase(iterator __position);
iterator
_M_erase(iterator __first, iterator __last);
private:
void
_M_move_assign(vector&& __x, true_type) noexcept
{
vector __tmp(get_allocator());
this->_M_impl._M_swap_data(__x._M_impl);
__tmp._M_impl._M_swap_data(__x._M_impl);
std::__alloc_on_move(_M_get_Tp_allocator(), __x._M_get_Tp_allocator());
}
void
_M_move_assign(vector&& __x, false_type)
{
if (__x._M_get_Tp_allocator() == this->_M_get_Tp_allocator())
_M_move_assign(std::move(__x), true_type());
else
{
this->_M_assign_aux(std::make_move_iterator(__x.begin()),
std::make_move_iterator(__x.end()),
std::random_access_iterator_tag());
__x.clear();
}
}
template<typename _Up>
_Up*
_M_data_ptr(_Up* __ptr) const noexcept
{ return __ptr; }
template<typename _Ptr>
typename std::pointer_traits<_Ptr>::element_type*
_M_data_ptr(_Ptr __ptr) const
{ return empty() ? nullptr : std::__to_address(__ptr); }
};
template<typename _InputIterator, typename _ValT
= typename iterator_traits<_InputIterator>::value_type,
typename _Allocator = allocator<_ValT>,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireAllocator<_Allocator>>
vector(_InputIterator, _InputIterator, _Allocator = _Allocator())
-> vector<_ValT, _Allocator>;
template<typename _Tp, typename _Alloc>
inline bool
operator==(const vector<_Tp, _Alloc>& __x, const vector<_Tp, _Alloc>& __y)
{ return (__x.size() == __y.size()
&& std::equal(__x.begin(), __x.end(), __y.begin())); }
template<typename _Tp, typename _Alloc>
inline bool
operator<(const vector<_Tp, _Alloc>& __x, const vector<_Tp, _Alloc>& __y)
{ return std::lexicographical_compare(__x.begin(), __x.end(),
__y.begin(), __y.end()); }
template<typename _Tp, typename _Alloc>
inline bool
operator!=(const vector<_Tp, _Alloc>& __x, const vector<_Tp, _Alloc>& __y)
{ return !(__x == __y); }
template<typename _Tp, typename _Alloc>
inline bool
operator>(const vector<_Tp, _Alloc>& __x, const vector<_Tp, _Alloc>& __y)
{ return __y < __x; }
template<typename _Tp, typename _Alloc>
inline bool
operator<=(const vector<_Tp, _Alloc>& __x, const vector<_Tp, _Alloc>& __y)
{ return !(__y < __x); }
template<typename _Tp, typename _Alloc>
inline bool
operator>=(const vector<_Tp, _Alloc>& __x, const vector<_Tp, _Alloc>& __y)
{ return !(__x < __y); }
template<typename _Tp, typename _Alloc>
inline void
swap(vector<_Tp, _Alloc>& __x, vector<_Tp, _Alloc>& __y)
noexcept(noexcept(__x.swap(__y)))
{ __x.swap(__y); }
namespace __detail::__variant
{
template<typename> struct _Never_valueless_alt;
template<typename _Tp, typename _Alloc>
struct _Never_valueless_alt<std::vector<_Tp, _Alloc>>
: std::is_nothrow_move_assignable<std::vector<_Tp, _Alloc>>
{ };
}
}
namespace std
{
size_t
_Hash_bytes(const void* __ptr, size_t __len, size_t __seed);
size_t
_Fnv_hash_bytes(const void* __ptr, size_t __len, size_t __seed);
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Result, typename _Arg>
struct __hash_base
{
typedef _Result result_type [[__deprecated__]];
typedef _Arg argument_type [[__deprecated__]];
};
template<typename _Tp>
struct hash;
template<typename _Tp, typename = void>
struct __poison_hash
{
static constexpr bool __enable_hash_call = false;
private:
__poison_hash(__poison_hash&&);
~__poison_hash();
};
template<typename _Tp>
struct __poison_hash<_Tp, __void_t<decltype(hash<_Tp>()(declval<_Tp>()))>>
{
static constexpr bool __enable_hash_call = true;
};
template<typename _Tp, bool = is_enum<_Tp>::value>
struct __hash_enum
{
private:
__hash_enum(__hash_enum&&);
~__hash_enum();
};
template<typename _Tp>
struct __hash_enum<_Tp, true> : public __hash_base<size_t, _Tp>
{
size_t
operator()(_Tp __val) const noexcept
{
using __type = typename underlying_type<_Tp>::type;
return hash<__type>{}(static_cast<__type>(__val));
}
};
template<typename _Tp>
struct hash : __hash_enum<_Tp>
{ };
template<typename _Tp>
struct hash<_Tp*> : public __hash_base<size_t, _Tp*>
{
size_t
operator()(_Tp* __p) const noexcept
{ return reinterpret_cast<size_t>(__p); }
};
template<> struct hash<bool> : public __hash_base<size_t, bool> { size_t operator()(bool __val) const noexcept { return static_cast<size_t>(__val); } };
template<> struct hash<char> : public __hash_base<size_t, char> { size_t operator()(char __val) const noexcept { return static_cast<size_t>(__val); } };
template<> struct hash<signed char> : public __hash_base<size_t, signed char> { size_t operator()(signed char __val) const noexcept { return static_cast<size_t>(__val); } };
template<> struct hash<unsigned char> : public __hash_base<size_t, unsigned char> { size_t operator()(unsigned char __val) const noexcept { return static_cast<size_t>(__val); } };
template<> struct hash<wchar_t> : public __hash_base<size_t, wchar_t> { size_t operator()(wchar_t __val) const noexcept { return static_cast<size_t>(__val); } };
template<> struct hash<char16_t> : public __hash_base<size_t, char16_t> { size_t operator()(char16_t __val) const noexcept { return static_cast<size_t>(__val); } };
template<> struct hash<char32_t> : public __hash_base<size_t, char32_t> { size_t operator()(char32_t __val) const noexcept { return static_cast<size_t>(__val); } };
template<> struct hash<short> : public __hash_base<size_t, short> { size_t operator()(short __val) const noexcept { return static_cast<size_t>(__val); } };
template<> struct hash<int> : public __hash_base<size_t, int> { size_t operator()(int __val) const noexcept { return static_cast<size_t>(__val); } };
template<> struct hash<long> : public __hash_base<size_t, long> { size_t operator()(long __val) const noexcept { return static_cast<size_t>(__val); } };
template<> struct hash<long long> : public __hash_base<size_t, long long> { size_t operator()(long long __val) const noexcept { return static_cast<size_t>(__val); } };
template<> struct hash<unsigned short> : public __hash_base<size_t, unsigned short> { size_t operator()(unsigned short __val) const noexcept { return static_cast<size_t>(__val); } };
template<> struct hash<unsigned int> : public __hash_base<size_t, unsigned int> { size_t operator()(unsigned int __val) const noexcept { return static_cast<size_t>(__val); } };
template<> struct hash<unsigned long> : public __hash_base<size_t, unsigned long> { size_t operator()(unsigned long __val) const noexcept { return static_cast<size_t>(__val); } };
template<> struct hash<unsigned long long> : public __hash_base<size_t, unsigned long long> { size_t operator()(unsigned long long __val) const noexcept { return static_cast<size_t>(__val); } };
struct _Hash_impl
{
static size_t
hash(const void* __ptr, size_t __clength,
size_t __seed = static_cast<size_t>(0xc70f6907UL))
{ return _Hash_bytes(__ptr, __clength, __seed); }
template<typename _Tp>
static size_t
hash(const _Tp& __val)
{ return hash(&__val, sizeof(__val)); }
template<typename _Tp>
static size_t
__hash_combine(const _Tp& __val, size_t __hash)
{ return hash(&__val, sizeof(__val), __hash); }
};
struct _Fnv_hash_impl
{
static size_t
hash(const void* __ptr, size_t __clength,
size_t __seed = static_cast<size_t>(2166136261UL))
{ return _Fnv_hash_bytes(__ptr, __clength, __seed); }
template<typename _Tp>
static size_t
hash(const _Tp& __val)
{ return hash(&__val, sizeof(__val)); }
template<typename _Tp>
static size_t
__hash_combine(const _Tp& __val, size_t __hash)
{ return hash(&__val, sizeof(__val), __hash); }
};
template<>
struct hash<float> : public __hash_base<size_t, float>
{
size_t
operator()(float __val) const noexcept
{
return __val != 0.0f ? std::_Hash_impl::hash(__val) : 0;
}
};
template<>
struct hash<double> : public __hash_base<size_t, double>
{
size_t
operator()(double __val) const noexcept
{
return __val != 0.0 ? std::_Hash_impl::hash(__val) : 0;
}
};
template<>
struct hash<long double>
: public __hash_base<size_t, long double>
{
__attribute__ ((__pure__)) size_t
operator()(long double __val) const noexcept;
};
template<>
struct hash<nullptr_t> : public __hash_base<size_t, nullptr_t>
{
size_t
operator()(nullptr_t) const noexcept
{ return 0; }
};
template<typename _Hash>
struct __is_fast_hash : public std::true_type
{ };
template<>
struct __is_fast_hash<hash<long double>> : public std::false_type
{ };
}
namespace std __attribute__ ((__visibility__ ("default")))
{
typedef unsigned long _Bit_type;
enum { _S_word_bit = int(8 * sizeof(_Bit_type)) };
__attribute__((__nonnull__))
void
__fill_bvector_n(_Bit_type*, size_t, bool) noexcept;
struct _Bit_reference
{
_Bit_type * _M_p;
_Bit_type _M_mask;
_Bit_reference(_Bit_type * __x, _Bit_type __y)
: _M_p(__x), _M_mask(__y) { }
_Bit_reference() noexcept : _M_p(0), _M_mask(0) { }
_Bit_reference(const _Bit_reference&) = default;
[[__nodiscard__]]
operator bool() const noexcept
{ return !!(*_M_p & _M_mask); }
_Bit_reference&
operator=(bool __x) noexcept
{
if (__x)
*_M_p |= _M_mask;
else
*_M_p &= ~_M_mask;
return *this;
}
_Bit_reference&
operator=(const _Bit_reference& __x) noexcept
{ return *this = bool(__x); }
[[__nodiscard__]]
bool
operator==(const _Bit_reference& __x) const
{ return bool(*this) == bool(__x); }
[[__nodiscard__]]
bool
operator<(const _Bit_reference& __x) const
{ return !bool(*this) && bool(__x); }
void
flip() noexcept
{ *_M_p ^= _M_mask; }
friend void
swap(_Bit_reference __x, _Bit_reference __y) noexcept
{
bool __tmp = __x;
__x = __y;
__y = __tmp;
}
friend void
swap(_Bit_reference __x, bool& __y) noexcept
{
bool __tmp = __x;
__x = __y;
__y = __tmp;
}
friend void
swap(bool& __x, _Bit_reference __y) noexcept
{
bool __tmp = __x;
__x = __y;
__y = __tmp;
}
};
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
struct _Bit_iterator_base
: public std::iterator<std::random_access_iterator_tag, bool>
{
_Bit_type * _M_p;
unsigned int _M_offset;
_Bit_iterator_base(_Bit_type * __x, unsigned int __y)
: _M_p(__x), _M_offset(__y) { }
void
_M_bump_up()
{
if (_M_offset++ == int(_S_word_bit) - 1)
{
_M_offset = 0;
++_M_p;
}
}
void
_M_bump_down()
{
if (_M_offset-- == 0)
{
_M_offset = int(_S_word_bit) - 1;
--_M_p;
}
}
void
_M_incr(ptrdiff_t __i)
{
difference_type __n = __i + _M_offset;
_M_p += __n / int(_S_word_bit);
__n = __n % int(_S_word_bit);
if (__n < 0)
{
__n += int(_S_word_bit);
--_M_p;
}
_M_offset = static_cast<unsigned int>(__n);
}
[[__nodiscard__]]
friend bool
operator==(const _Bit_iterator_base& __x, const _Bit_iterator_base& __y)
{ return __x._M_p == __y._M_p && __x._M_offset == __y._M_offset; }
[[__nodiscard__]]
friend bool
operator<(const _Bit_iterator_base& __x, const _Bit_iterator_base& __y)
{
return __x._M_p < __y._M_p
|| (__x._M_p == __y._M_p && __x._M_offset < __y._M_offset);
}
[[__nodiscard__]]
friend bool
operator!=(const _Bit_iterator_base& __x, const _Bit_iterator_base& __y)
{ return !(__x == __y); }
[[__nodiscard__]]
friend bool
operator>(const _Bit_iterator_base& __x, const _Bit_iterator_base& __y)
{ return __y < __x; }
[[__nodiscard__]]
friend bool
operator<=(const _Bit_iterator_base& __x, const _Bit_iterator_base& __y)
{ return !(__y < __x); }
[[__nodiscard__]]
friend bool
operator>=(const _Bit_iterator_base& __x, const _Bit_iterator_base& __y)
{ return !(__x < __y); }
friend ptrdiff_t
operator-(const _Bit_iterator_base& __x, const _Bit_iterator_base& __y)
{
return (int(_S_word_bit) * (__x._M_p - __y._M_p)
+ __x._M_offset - __y._M_offset);
}
};
#pragma GCC diagnostic pop
struct _Bit_iterator : public _Bit_iterator_base
{
typedef _Bit_reference reference;
typedef _Bit_reference* pointer;
typedef _Bit_iterator iterator;
_Bit_iterator() : _Bit_iterator_base(0, 0) { }
_Bit_iterator(_Bit_type * __x, unsigned int __y)
: _Bit_iterator_base(__x, __y) { }
iterator
_M_const_cast() const
{ return *this; }
[[__nodiscard__]]
reference
operator*() const
{ return reference(_M_p, 1UL << _M_offset); }
iterator&
operator++()
{
_M_bump_up();
return *this;
}
iterator
operator++(int)
{
iterator __tmp = *this;
_M_bump_up();
return __tmp;
}
iterator&
operator--()
{
_M_bump_down();
return *this;
}
iterator
operator--(int)
{
iterator __tmp = *this;
_M_bump_down();
return __tmp;
}
iterator&
operator+=(difference_type __i)
{
_M_incr(__i);
return *this;
}
iterator&
operator-=(difference_type __i)
{
*this += -__i;
return *this;
}
[[__nodiscard__]]
reference
operator[](difference_type __i) const
{ return *(*this + __i); }
[[__nodiscard__]]
friend iterator
operator+(const iterator& __x, difference_type __n)
{
iterator __tmp = __x;
__tmp += __n;
return __tmp;
}
[[__nodiscard__]]
friend iterator
operator+(difference_type __n, const iterator& __x)
{ return __x + __n; }
[[__nodiscard__]]
friend iterator
operator-(const iterator& __x, difference_type __n)
{
iterator __tmp = __x;
__tmp -= __n;
return __tmp;
}
};
struct _Bit_const_iterator : public _Bit_iterator_base
{
typedef bool reference;
typedef bool const_reference;
typedef const bool* pointer;
typedef _Bit_const_iterator const_iterator;
_Bit_const_iterator() : _Bit_iterator_base(0, 0) { }
_Bit_const_iterator(_Bit_type * __x, unsigned int __y)
: _Bit_iterator_base(__x, __y) { }
_Bit_const_iterator(const _Bit_iterator& __x)
: _Bit_iterator_base(__x._M_p, __x._M_offset) { }
_Bit_iterator
_M_const_cast() const
{ return _Bit_iterator(_M_p, _M_offset); }
[[__nodiscard__]]
const_reference
operator*() const
{ return _Bit_reference(_M_p, 1UL << _M_offset); }
const_iterator&
operator++()
{
_M_bump_up();
return *this;
}
const_iterator
operator++(int)
{
const_iterator __tmp = *this;
_M_bump_up();
return __tmp;
}
const_iterator&
operator--()
{
_M_bump_down();
return *this;
}
const_iterator
operator--(int)
{
const_iterator __tmp = *this;
_M_bump_down();
return __tmp;
}
const_iterator&
operator+=(difference_type __i)
{
_M_incr(__i);
return *this;
}
const_iterator&
operator-=(difference_type __i)
{
*this += -__i;
return *this;
}
[[__nodiscard__]]
const_reference
operator[](difference_type __i) const
{ return *(*this + __i); }
[[__nodiscard__]]
friend const_iterator
operator+(const const_iterator& __x, difference_type __n)
{
const_iterator __tmp = __x;
__tmp += __n;
return __tmp;
}
[[__nodiscard__]]
friend const_iterator
operator-(const const_iterator& __x, difference_type __n)
{
const_iterator __tmp = __x;
__tmp -= __n;
return __tmp;
}
[[__nodiscard__]]
friend const_iterator
operator+(difference_type __n, const const_iterator& __x)
{ return __x + __n; }
};
template<typename _Alloc>
struct _Bvector_base
{
typedef typename __gnu_cxx::__alloc_traits<_Alloc>::template
rebind<_Bit_type>::other _Bit_alloc_type;
typedef typename __gnu_cxx::__alloc_traits<_Bit_alloc_type>
_Bit_alloc_traits;
typedef typename _Bit_alloc_traits::pointer _Bit_pointer;
struct _Bvector_impl_data
{
_Bit_iterator _M_start;
_Bit_iterator _M_finish;
_Bit_pointer _M_end_of_storage;
_Bvector_impl_data() noexcept
: _M_start(), _M_finish(), _M_end_of_storage()
{ }
_Bvector_impl_data(const _Bvector_impl_data&) = default;
_Bvector_impl_data&
operator=(const _Bvector_impl_data&) = default;
_Bvector_impl_data(_Bvector_impl_data&& __x) noexcept
: _Bvector_impl_data(__x)
{ __x._M_reset(); }
void
_M_move_data(_Bvector_impl_data&& __x) noexcept
{
*this = __x;
__x._M_reset();
}
void
_M_reset() noexcept
{ *this = _Bvector_impl_data(); }
void
_M_swap_data(_Bvector_impl_data& __x) noexcept
{
std::swap(*this, __x);
}
};
struct _Bvector_impl
: public _Bit_alloc_type, public _Bvector_impl_data
{
_Bvector_impl() noexcept(is_nothrow_default_constructible<_Bit_alloc_type>::value)
: _Bit_alloc_type()
{ }
_Bvector_impl(const _Bit_alloc_type& __a) noexcept
: _Bit_alloc_type(__a)
{ }
_Bvector_impl(_Bvector_impl&& __x) noexcept
: _Bit_alloc_type(std::move(__x)), _Bvector_impl_data(std::move(__x))
{ }
_Bvector_impl(_Bit_alloc_type&& __a, _Bvector_impl&& __x) noexcept
: _Bit_alloc_type(std::move(__a)), _Bvector_impl_data(std::move(__x))
{ }
_Bit_type*
_M_end_addr() const noexcept
{
if (this->_M_end_of_storage)
return std::__addressof(this->_M_end_of_storage[-1]) + 1;
return 0;
}
};
public:
typedef _Alloc allocator_type;
_Bit_alloc_type&
_M_get_Bit_allocator() noexcept
{ return this->_M_impl; }
const _Bit_alloc_type&
_M_get_Bit_allocator() const noexcept
{ return this->_M_impl; }
allocator_type
get_allocator() const noexcept
{ return allocator_type(_M_get_Bit_allocator()); }
_Bvector_base() = default;
_Bvector_base(const allocator_type& __a)
: _M_impl(__a) { }
_Bvector_base(_Bvector_base&&) = default;
_Bvector_base(_Bvector_base&& __x, const allocator_type& __a) noexcept
: _M_impl(_Bit_alloc_type(__a), std::move(__x._M_impl))
{ }
~_Bvector_base()
{ this->_M_deallocate(); }
protected:
_Bvector_impl _M_impl;
_Bit_pointer
_M_allocate(size_t __n)
{
_Bit_pointer __p = _Bit_alloc_traits::allocate(_M_impl, _S_nword(__n));
return __p;
}
void
_M_deallocate()
{
if (_M_impl._M_start._M_p)
{
const size_t __n = _M_impl._M_end_addr() - _M_impl._M_start._M_p;
_Bit_alloc_traits::deallocate(_M_impl,
_M_impl._M_end_of_storage - __n,
__n);
_M_impl._M_reset();
}
}
void
_M_move_data(_Bvector_base&& __x) noexcept
{ _M_impl._M_move_data(std::move(__x._M_impl)); }
constexpr
static size_t
_S_nword(size_t __n)
{ return (__n + int(_S_word_bit) - 1) / int(_S_word_bit); }
};
template<typename _Alloc>
class vector<bool, _Alloc> : protected _Bvector_base<_Alloc>
{
typedef _Bvector_base<_Alloc> _Base;
typedef typename _Base::_Bit_pointer _Bit_pointer;
typedef typename _Base::_Bit_alloc_traits _Bit_alloc_traits;
friend struct std::hash<vector>;
public:
typedef bool value_type;
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef _Bit_reference reference;
typedef bool const_reference;
typedef _Bit_reference* pointer;
typedef const bool* const_pointer;
typedef _Bit_iterator iterator;
typedef _Bit_const_iterator const_iterator;
typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
typedef std::reverse_iterator<iterator> reverse_iterator;
typedef _Alloc allocator_type;
allocator_type
get_allocator() const
{ return _Base::get_allocator(); }
protected:
using _Base::_M_allocate;
using _Base::_M_deallocate;
using _Base::_S_nword;
using _Base::_M_get_Bit_allocator;
public:
vector() = default;
explicit
vector(const allocator_type& __a)
: _Base(__a) { }
explicit
vector(size_type __n, const allocator_type& __a = allocator_type())
: vector(__n, false, __a)
{ }
vector(size_type __n, const bool& __value,
const allocator_type& __a = allocator_type())
: _Base(__a)
{
_M_initialize(__n);
_M_initialize_value(__value);
}
vector(const vector& __x)
: _Base(_Bit_alloc_traits::_S_select_on_copy(__x._M_get_Bit_allocator()))
{
const_iterator __xbegin = __x.begin(), __xend = __x.end();
_M_initialize(__x.size());
_M_copy_aligned(__xbegin, __xend, begin());
}
vector(vector&&) = default;
private:
vector(vector&& __x, const allocator_type& __a, true_type) noexcept
: _Base(std::move(__x), __a)
{ }
vector(vector&& __x, const allocator_type& __a, false_type)
: _Base(__a)
{
if (__x.get_allocator() == __a)
this->_M_move_data(std::move(__x));
else
{
_M_initialize(__x.size());
_M_copy_aligned(__x.begin(), __x.end(), begin());
__x.clear();
}
}
public:
vector(vector&& __x, const __type_identity_t<allocator_type>& __a)
noexcept(_Bit_alloc_traits::_S_always_equal())
: vector(std::move(__x), __a,
typename _Bit_alloc_traits::is_always_equal{})
{ }
vector(const vector& __x, const __type_identity_t<allocator_type>& __a)
: _Base(__a)
{
_M_initialize(__x.size());
_M_copy_aligned(__x.begin(), __x.end(), begin());
}
vector(initializer_list<bool> __l,
const allocator_type& __a = allocator_type())
: _Base(__a)
{
_M_initialize_range(__l.begin(), __l.end(),
random_access_iterator_tag());
}
template<typename _InputIterator,
typename = std::_RequireInputIter<_InputIterator>>
vector(_InputIterator __first, _InputIterator __last,
const allocator_type& __a = allocator_type())
: _Base(__a)
{
_M_initialize_range(__first, __last,
std::__iterator_category(__first));
}
~vector() noexcept { }
vector&
operator=(const vector& __x)
{
if (&__x == this)
return *this;
if (_Bit_alloc_traits::_S_propagate_on_copy_assign())
{
if (this->_M_get_Bit_allocator() != __x._M_get_Bit_allocator())
{
this->_M_deallocate();
std::__alloc_on_copy(_M_get_Bit_allocator(),
__x._M_get_Bit_allocator());
_M_initialize(__x.size());
}
else
std::__alloc_on_copy(_M_get_Bit_allocator(),
__x._M_get_Bit_allocator());
}
if (__x.size() > capacity())
{
this->_M_deallocate();
_M_initialize(__x.size());
}
this->_M_impl._M_finish = _M_copy_aligned(__x.begin(), __x.end(),
begin());
return *this;
}
vector&
operator=(vector&& __x) noexcept(_Bit_alloc_traits::_S_nothrow_move())
{
if (_Bit_alloc_traits::_S_propagate_on_move_assign()
|| this->_M_get_Bit_allocator() == __x._M_get_Bit_allocator())
{
this->_M_deallocate();
this->_M_move_data(std::move(__x));
std::__alloc_on_move(_M_get_Bit_allocator(),
__x._M_get_Bit_allocator());
}
else
{
if (__x.size() > capacity())
{
this->_M_deallocate();
_M_initialize(__x.size());
}
this->_M_impl._M_finish = _M_copy_aligned(__x.begin(), __x.end(),
begin());
__x.clear();
}
return *this;
}
vector&
operator=(initializer_list<bool> __l)
{
this->assign(__l.begin(), __l.end());
return *this;
}
void
assign(size_type __n, const bool& __x)
{ _M_fill_assign(__n, __x); }
template<typename _InputIterator,
typename = std::_RequireInputIter<_InputIterator>>
void
assign(_InputIterator __first, _InputIterator __last)
{ _M_assign_aux(__first, __last, std::__iterator_category(__first)); }
void
assign(initializer_list<bool> __l)
{ _M_assign_aux(__l.begin(), __l.end(), random_access_iterator_tag()); }
[[__nodiscard__]]
iterator
begin() noexcept
{ return iterator(this->_M_impl._M_start._M_p, 0); }
[[__nodiscard__]]
const_iterator
begin() const noexcept
{ return const_iterator(this->_M_impl._M_start._M_p, 0); }
[[__nodiscard__]]
iterator
end() noexcept
{ return this->_M_impl._M_finish; }
[[__nodiscard__]]
const_iterator
end() const noexcept
{ return this->_M_impl._M_finish; }
[[__nodiscard__]]
reverse_iterator
rbegin() noexcept
{ return reverse_iterator(end()); }
[[__nodiscard__]]
const_reverse_iterator
rbegin() const noexcept
{ return const_reverse_iterator(end()); }
[[__nodiscard__]]
reverse_iterator
rend() noexcept
{ return reverse_iterator(begin()); }
[[__nodiscard__]]
const_reverse_iterator
rend() const noexcept
{ return const_reverse_iterator(begin()); }
[[__nodiscard__]]
const_iterator
cbegin() const noexcept
{ return const_iterator(this->_M_impl._M_start._M_p, 0); }
[[__nodiscard__]]
const_iterator
cend() const noexcept
{ return this->_M_impl._M_finish; }
[[__nodiscard__]]
const_reverse_iterator
crbegin() const noexcept
{ return const_reverse_iterator(end()); }
[[__nodiscard__]]
const_reverse_iterator
crend() const noexcept
{ return const_reverse_iterator(begin()); }
[[__nodiscard__]]
size_type
size() const noexcept
{ return size_type(end() - begin()); }
[[__nodiscard__]]
size_type
max_size() const noexcept
{
const size_type __isize =
__gnu_cxx::__numeric_traits<difference_type>::__max
- int(_S_word_bit) + 1;
const size_type __asize
= _Bit_alloc_traits::max_size(_M_get_Bit_allocator());
return (__asize <= __isize / int(_S_word_bit)
? __asize * int(_S_word_bit) : __isize);
}
[[__nodiscard__]]
size_type
capacity() const noexcept
{ return size_type(const_iterator(this->_M_impl._M_end_addr(), 0)
- begin()); }
[[__nodiscard__]]
bool
empty() const noexcept
{ return begin() == end(); }
[[__nodiscard__]]
reference
operator[](size_type __n)
{ return begin()[__n]; }
[[__nodiscard__]]
const_reference
operator[](size_type __n) const
{ return begin()[__n]; }
protected:
void
_M_range_check(size_type __n) const
{
if (__n >= this->size())
__throw_out_of_range_fmt(("vector<bool>::_M_range_check: __n " "(which is %zu) >= this->size() " "(which is %zu)")
,
__n, this->size());
}
public:
reference
at(size_type __n)
{
_M_range_check(__n);
return (*this)[__n];
}
const_reference
at(size_type __n) const
{
_M_range_check(__n);
return (*this)[__n];
}
void
reserve(size_type __n)
{
if (__n > max_size())
__throw_length_error(("vector::reserve"));
if (capacity() < __n)
_M_reallocate(__n);
}
[[__nodiscard__]]
reference
front()
{ return *begin(); }
[[__nodiscard__]]
const_reference
front() const
{ return *begin(); }
[[__nodiscard__]]
reference
back()
{ return *(end() - 1); }
[[__nodiscard__]]
const_reference
back() const
{ return *(end() - 1); }
void
push_back(bool __x)
{
if (this->_M_impl._M_finish._M_p != this->_M_impl._M_end_addr())
*this->_M_impl._M_finish++ = __x;
else
_M_insert_aux(end(), __x);
}
void
swap(vector& __x) noexcept
{
do { if (std::__is_constant_evaluated() && !bool(_Bit_alloc_traits::propagate_on_container_swap::value || _M_get_Bit_allocator() == __x._M_get_Bit_allocator())) __builtin_unreachable(); } while (false)
;
this->_M_impl._M_swap_data(__x._M_impl);
_Bit_alloc_traits::_S_on_swap(_M_get_Bit_allocator(),
__x._M_get_Bit_allocator());
}
static void
swap(reference __x, reference __y) noexcept
{
bool __tmp = __x;
__x = __y;
__y = __tmp;
}
iterator
insert(const_iterator __position, const bool& __x)
{
const difference_type __n = __position - begin();
if (this->_M_impl._M_finish._M_p != this->_M_impl._M_end_addr()
&& __position == end())
*this->_M_impl._M_finish++ = __x;
else
_M_insert_aux(__position._M_const_cast(), __x);
return begin() + __n;
}
__attribute__ ((__deprecated__ ("use '" "insert(position, false)" "' instead")))
iterator
insert(const_iterator __position)
{ return this->insert(__position._M_const_cast(), false); }
template<typename _InputIterator,
typename = std::_RequireInputIter<_InputIterator>>
iterator
insert(const_iterator __position,
_InputIterator __first, _InputIterator __last)
{
difference_type __offset = __position - cbegin();
_M_insert_range(__position._M_const_cast(),
__first, __last,
std::__iterator_category(__first));
return begin() + __offset;
}
iterator
insert(const_iterator __position, size_type __n, const bool& __x)
{
difference_type __offset = __position - cbegin();
_M_fill_insert(__position._M_const_cast(), __n, __x);
return begin() + __offset;
}
iterator
insert(const_iterator __p, initializer_list<bool> __l)
{ return this->insert(__p, __l.begin(), __l.end()); }
void
pop_back()
{ --this->_M_impl._M_finish; }
iterator
erase(const_iterator __position)
{ return _M_erase(__position._M_const_cast()); }
iterator
erase(const_iterator __first, const_iterator __last)
{ return _M_erase(__first._M_const_cast(), __last._M_const_cast()); }
void
resize(size_type __new_size, bool __x = bool())
{
if (__new_size < size())
_M_erase_at_end(begin() + difference_type(__new_size));
else
insert(end(), __new_size - size(), __x);
}
void
shrink_to_fit()
{ _M_shrink_to_fit(); }
void
flip() noexcept
{
_Bit_type * const __end = this->_M_impl._M_end_addr();
for (_Bit_type * __p = this->_M_impl._M_start._M_p; __p != __end; ++__p)
*__p = ~*__p;
}
void
clear() noexcept
{ _M_erase_at_end(begin()); }
template<typename... _Args>
reference
emplace_back(_Args&&... __args)
{
push_back(bool(__args...));
return back();
}
template<typename... _Args>
iterator
emplace(const_iterator __pos, _Args&&... __args)
{ return insert(__pos, bool(__args...)); }
protected:
iterator
_M_copy_aligned(const_iterator __first, const_iterator __last,
iterator __result)
{
_Bit_type* __q = std::copy(__first._M_p, __last._M_p, __result._M_p);
return std::copy(const_iterator(__last._M_p, 0), __last,
iterator(__q, 0));
}
void
_M_initialize(size_type __n)
{
if (__n)
{
_Bit_pointer __q = this->_M_allocate(__n);
this->_M_impl._M_end_of_storage = __q + _S_nword(__n);
iterator __start = iterator(std::__addressof(*__q), 0);
this->_M_impl._M_start = __start;
this->_M_impl._M_finish = __start + difference_type(__n);
}
}
void
_M_initialize_value(bool __x) noexcept
{
if (_Bit_type* __p = this->_M_impl._M_start._M_p)
__fill_bvector_n(__p, this->_M_impl._M_end_addr() - __p, __x);
}
void
_M_reallocate(size_type __n);
bool
_M_shrink_to_fit();
template<typename _InputIterator>
void
_M_initialize_range(_InputIterator __first, _InputIterator __last,
std::input_iterator_tag)
{
for (; __first != __last; ++__first)
push_back(*__first);
}
template<typename _ForwardIterator>
void
_M_initialize_range(_ForwardIterator __first, _ForwardIterator __last,
std::forward_iterator_tag)
{
const size_type __n = std::distance(__first, __last);
_M_initialize(__n);
std::copy(__first, __last, begin());
}
void
_M_fill_assign(size_t __n, bool __x)
{
if (__n > size())
{
_M_initialize_value(__x);
insert(end(), __n - size(), __x);
}
else
{
_M_erase_at_end(begin() + __n);
_M_initialize_value(__x);
}
}
template<typename _InputIterator>
void
_M_assign_aux(_InputIterator __first, _InputIterator __last,
std::input_iterator_tag)
{
iterator __cur = begin();
for (; __first != __last && __cur != end(); ++__cur, (void)++__first)
*__cur = *__first;
if (__first == __last)
_M_erase_at_end(__cur);
else
insert(end(), __first, __last);
}
template<typename _ForwardIterator>
void
_M_assign_aux(_ForwardIterator __first, _ForwardIterator __last,
std::forward_iterator_tag)
{
const size_type __len = std::distance(__first, __last);
if (__len < size())
_M_erase_at_end(std::copy(__first, __last, begin()));
else
{
_ForwardIterator __mid = __first;
std::advance(__mid, size());
std::copy(__first, __mid, begin());
insert(end(), __mid, __last);
}
}
void
_M_fill_insert(iterator __position, size_type __n, bool __x);
template<typename _InputIterator>
void
_M_insert_range(iterator __pos, _InputIterator __first,
_InputIterator __last, std::input_iterator_tag)
{
for (; __first != __last; ++__first)
{
__pos = insert(__pos, *__first);
++__pos;
}
}
template<typename _ForwardIterator>
void
_M_insert_range(iterator __position, _ForwardIterator __first,
_ForwardIterator __last, std::forward_iterator_tag);
void
_M_insert_aux(iterator __position, bool __x);
size_type
_M_check_len(size_type __n, const char* __s) const
{
if (max_size() - size() < __n)
__throw_length_error((__s));
const size_type __len = size() + std::max(size(), __n);
return (__len < size() || __len > max_size()) ? max_size() : __len;
}
void
_M_erase_at_end(iterator __pos)
{ this->_M_impl._M_finish = __pos; }
iterator
_M_erase(iterator __pos);
iterator
_M_erase(iterator __first, iterator __last);
protected:
void data() = delete;
};
inline void
__fill_bvector(_Bit_type* __v, unsigned int __first, unsigned int __last,
bool __x) noexcept
{
const _Bit_type __fmask = ~0ul << __first;
const _Bit_type __lmask = ~0ul >> (_S_word_bit - __last);
const _Bit_type __mask = __fmask & __lmask;
if (__x)
*__v |= __mask;
else
*__v &= ~__mask;
}
__attribute__((__nonnull__))
inline void
__fill_bvector_n(_Bit_type* __p, size_t __n, bool __x) noexcept
{
__builtin_memset(__p, __x ? ~0 : 0, __n * sizeof(_Bit_type));
}
inline void
__fill_a1(std::_Bit_iterator __first,
std::_Bit_iterator __last, const bool& __x)
{
if (__first._M_p != __last._M_p)
{
_Bit_type* __first_p = __first._M_p;
if (__first._M_offset != 0)
__fill_bvector(__first_p++, __first._M_offset, _S_word_bit, __x);
__fill_bvector_n(__first_p, __last._M_p - __first_p, __x);
if (__last._M_offset != 0)
__fill_bvector(__last._M_p, 0, __last._M_offset, __x);
}
else if (__first._M_offset != __last._M_offset)
__fill_bvector(__first._M_p, __first._M_offset, __last._M_offset, __x);
}
template<typename _Alloc>
struct hash<std::vector<bool, _Alloc>>
: public __hash_base<size_t, std::vector<bool, _Alloc>>
{
size_t
operator()(const std::vector<bool, _Alloc>&) const noexcept;
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Tp, typename _Up = typename __inv_unwrap<_Tp>::type>
constexpr _Up&&
__invfwd(typename remove_reference<_Tp>::type& __t) noexcept
{ return static_cast<_Up&&>(__t); }
template<typename _Res, typename _Fn, typename... _Args>
constexpr _Res
__invoke_impl(__invoke_other, _Fn&& __f, _Args&&... __args)
{ return std::forward<_Fn>(__f)(std::forward<_Args>(__args)...); }
template<typename _Res, typename _MemFun, typename _Tp, typename... _Args>
constexpr _Res
__invoke_impl(__invoke_memfun_ref, _MemFun&& __f, _Tp&& __t,
_Args&&... __args)
{ return (__invfwd<_Tp>(__t).*__f)(std::forward<_Args>(__args)...); }
template<typename _Res, typename _MemFun, typename _Tp, typename... _Args>
constexpr _Res
__invoke_impl(__invoke_memfun_deref, _MemFun&& __f, _Tp&& __t,
_Args&&... __args)
{
return ((*std::forward<_Tp>(__t)).*__f)(std::forward<_Args>(__args)...);
}
template<typename _Res, typename _MemPtr, typename _Tp>
constexpr _Res
__invoke_impl(__invoke_memobj_ref, _MemPtr&& __f, _Tp&& __t)
{ return __invfwd<_Tp>(__t).*__f; }
template<typename _Res, typename _MemPtr, typename _Tp>
constexpr _Res
__invoke_impl(__invoke_memobj_deref, _MemPtr&& __f, _Tp&& __t)
{ return (*std::forward<_Tp>(__t)).*__f; }
template<typename _Callable, typename... _Args>
constexpr typename __invoke_result<_Callable, _Args...>::type
__invoke(_Callable&& __fn, _Args&&... __args)
noexcept(__is_nothrow_invocable<_Callable, _Args...>::value)
{
using __result = __invoke_result<_Callable, _Args...>;
using __type = typename __result::type;
using __tag = typename __result::__invoke_type;
return std::__invoke_impl<__type>(__tag{}, std::forward<_Callable>(__fn),
std::forward<_Args>(__args)...);
}
template<typename _Res, typename _Callable, typename... _Args>
constexpr enable_if_t<is_invocable_r_v<_Res, _Callable, _Args...>, _Res>
__invoke_r(_Callable&& __fn, _Args&&... __args)
noexcept(is_nothrow_invocable_r_v<_Res, _Callable, _Args...>)
{
using __result = __invoke_result<_Callable, _Args...>;
using __type = typename __result::type;
using __tag = typename __result::__invoke_type;
if constexpr (is_void_v<_Res>)
std::__invoke_impl<__type>(__tag{}, std::forward<_Callable>(__fn),
std::forward<_Args>(__args)...);
else
return std::__invoke_impl<__type>(__tag{},
std::forward<_Callable>(__fn),
std::forward<_Args>(__args)...);
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Arg, typename _Result>
struct unary_function
{
typedef _Arg argument_type;
typedef _Result result_type;
} __attribute__ ((__deprecated__));
template<typename _Arg1, typename _Arg2, typename _Result>
struct binary_function
{
typedef _Arg1 first_argument_type;
typedef _Arg2 second_argument_type;
typedef _Result result_type;
} __attribute__ ((__deprecated__));
struct __is_transparent;
template<typename _Tp = void>
struct plus;
template<typename _Tp = void>
struct minus;
template<typename _Tp = void>
struct multiplies;
template<typename _Tp = void>
struct divides;
template<typename _Tp = void>
struct modulus;
template<typename _Tp = void>
struct negate;
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Tp>
struct plus : public binary_function<_Tp, _Tp, _Tp>
{
constexpr
_Tp
operator()(const _Tp& __x, const _Tp& __y) const
{ return __x + __y; }
};
template<typename _Tp>
struct minus : public binary_function<_Tp, _Tp, _Tp>
{
constexpr
_Tp
operator()(const _Tp& __x, const _Tp& __y) const
{ return __x - __y; }
};
template<typename _Tp>
struct multiplies : public binary_function<_Tp, _Tp, _Tp>
{
constexpr
_Tp
operator()(const _Tp& __x, const _Tp& __y) const
{ return __x * __y; }
};
template<typename _Tp>
struct divides : public binary_function<_Tp, _Tp, _Tp>
{
constexpr
_Tp
operator()(const _Tp& __x, const _Tp& __y) const
{ return __x / __y; }
};
template<typename _Tp>
struct modulus : public binary_function<_Tp, _Tp, _Tp>
{
constexpr
_Tp
operator()(const _Tp& __x, const _Tp& __y) const
{ return __x % __y; }
};
template<typename _Tp>
struct negate : public unary_function<_Tp, _Tp>
{
constexpr
_Tp
operator()(const _Tp& __x) const
{ return -__x; }
};
#pragma GCC diagnostic pop
template<>
struct plus<void>
{
template <typename _Tp, typename _Up>
constexpr
auto
operator()(_Tp&& __t, _Up&& __u) const
noexcept(noexcept(std::forward<_Tp>(__t) + std::forward<_Up>(__u)))
-> decltype(std::forward<_Tp>(__t) + std::forward<_Up>(__u))
{ return std::forward<_Tp>(__t) + std::forward<_Up>(__u); }
typedef __is_transparent is_transparent;
};
template<>
struct minus<void>
{
template <typename _Tp, typename _Up>
constexpr
auto
operator()(_Tp&& __t, _Up&& __u) const
noexcept(noexcept(std::forward<_Tp>(__t) - std::forward<_Up>(__u)))
-> decltype(std::forward<_Tp>(__t) - std::forward<_Up>(__u))
{ return std::forward<_Tp>(__t) - std::forward<_Up>(__u); }
typedef __is_transparent is_transparent;
};
template<>
struct multiplies<void>
{
template <typename _Tp, typename _Up>
constexpr
auto
operator()(_Tp&& __t, _Up&& __u) const
noexcept(noexcept(std::forward<_Tp>(__t) * std::forward<_Up>(__u)))
-> decltype(std::forward<_Tp>(__t) * std::forward<_Up>(__u))
{ return std::forward<_Tp>(__t) * std::forward<_Up>(__u); }
typedef __is_transparent is_transparent;
};
template<>
struct divides<void>
{
template <typename _Tp, typename _Up>
constexpr
auto
operator()(_Tp&& __t, _Up&& __u) const
noexcept(noexcept(std::forward<_Tp>(__t) / std::forward<_Up>(__u)))
-> decltype(std::forward<_Tp>(__t) / std::forward<_Up>(__u))
{ return std::forward<_Tp>(__t) / std::forward<_Up>(__u); }
typedef __is_transparent is_transparent;
};
template<>
struct modulus<void>
{
template <typename _Tp, typename _Up>
constexpr
auto
operator()(_Tp&& __t, _Up&& __u) const
noexcept(noexcept(std::forward<_Tp>(__t) % std::forward<_Up>(__u)))
-> decltype(std::forward<_Tp>(__t) % std::forward<_Up>(__u))
{ return std::forward<_Tp>(__t) % std::forward<_Up>(__u); }
typedef __is_transparent is_transparent;
};
template<>
struct negate<void>
{
template <typename _Tp>
constexpr
auto
operator()(_Tp&& __t) const
noexcept(noexcept(-std::forward<_Tp>(__t)))
-> decltype(-std::forward<_Tp>(__t))
{ return -std::forward<_Tp>(__t); }
typedef __is_transparent is_transparent;
};
template<typename _Tp = void>
struct equal_to;
template<typename _Tp = void>
struct not_equal_to;
template<typename _Tp = void>
struct greater;
template<typename _Tp = void>
struct less;
template<typename _Tp = void>
struct greater_equal;
template<typename _Tp = void>
struct less_equal;
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Tp>
struct equal_to : public binary_function<_Tp, _Tp, bool>
{
constexpr
bool
operator()(const _Tp& __x, const _Tp& __y) const
{ return __x == __y; }
};
template<typename _Tp>
struct not_equal_to : public binary_function<_Tp, _Tp, bool>
{
constexpr
bool
operator()(const _Tp& __x, const _Tp& __y) const
{ return __x != __y; }
};
template<typename _Tp>
struct greater : public binary_function<_Tp, _Tp, bool>
{
constexpr
bool
operator()(const _Tp& __x, const _Tp& __y) const
{ return __x > __y; }
};
template<typename _Tp>
struct less : public binary_function<_Tp, _Tp, bool>
{
constexpr
bool
operator()(const _Tp& __x, const _Tp& __y) const
{ return __x < __y; }
};
template<typename _Tp>
struct greater_equal : public binary_function<_Tp, _Tp, bool>
{
constexpr
bool
operator()(const _Tp& __x, const _Tp& __y) const
{ return __x >= __y; }
};
template<typename _Tp>
struct less_equal : public binary_function<_Tp, _Tp, bool>
{
constexpr
bool
operator()(const _Tp& __x, const _Tp& __y) const
{ return __x <= __y; }
};
template<typename _Tp>
struct greater<_Tp*> : public binary_function<_Tp*, _Tp*, bool>
{
constexpr bool
operator()(_Tp* __x, _Tp* __y) const noexcept
{
if (std::__is_constant_evaluated())
return __x > __y;
return (unsigned int)__x > (unsigned int)__y;
}
};
template<typename _Tp>
struct less<_Tp*> : public binary_function<_Tp*, _Tp*, bool>
{
constexpr bool
operator()(_Tp* __x, _Tp* __y) const noexcept
{
if (std::__is_constant_evaluated())
return __x < __y;
return (unsigned int)__x < (unsigned int)__y;
}
};
template<typename _Tp>
struct greater_equal<_Tp*> : public binary_function<_Tp*, _Tp*, bool>
{
constexpr bool
operator()(_Tp* __x, _Tp* __y) const noexcept
{
if (std::__is_constant_evaluated())
return __x >= __y;
return (unsigned int)__x >= (unsigned int)__y;
}
};
template<typename _Tp>
struct less_equal<_Tp*> : public binary_function<_Tp*, _Tp*, bool>
{
constexpr bool
operator()(_Tp* __x, _Tp* __y) const noexcept
{
if (std::__is_constant_evaluated())
return __x <= __y;
return (unsigned int)__x <= (unsigned int)__y;
}
};
#pragma GCC diagnostic pop
template<>
struct equal_to<void>
{
template <typename _Tp, typename _Up>
constexpr auto
operator()(_Tp&& __t, _Up&& __u) const
noexcept(noexcept(std::forward<_Tp>(__t) == std::forward<_Up>(__u)))
-> decltype(std::forward<_Tp>(__t) == std::forward<_Up>(__u))
{ return std::forward<_Tp>(__t) == std::forward<_Up>(__u); }
typedef __is_transparent is_transparent;
};
template<>
struct not_equal_to<void>
{
template <typename _Tp, typename _Up>
constexpr auto
operator()(_Tp&& __t, _Up&& __u) const
noexcept(noexcept(std::forward<_Tp>(__t) != std::forward<_Up>(__u)))
-> decltype(std::forward<_Tp>(__t) != std::forward<_Up>(__u))
{ return std::forward<_Tp>(__t) != std::forward<_Up>(__u); }
typedef __is_transparent is_transparent;
};
template<>
struct greater<void>
{
template <typename _Tp, typename _Up>
constexpr auto
operator()(_Tp&& __t, _Up&& __u) const
noexcept(noexcept(std::forward<_Tp>(__t) > std::forward<_Up>(__u)))
-> decltype(std::forward<_Tp>(__t) > std::forward<_Up>(__u))
{
return _S_cmp(std::forward<_Tp>(__t), std::forward<_Up>(__u),
__ptr_cmp<_Tp, _Up>{});
}
template<typename _Tp, typename _Up>
constexpr bool
operator()(_Tp* __t, _Up* __u) const noexcept
{ return greater<common_type_t<_Tp*, _Up*>>{}(__t, __u); }
typedef __is_transparent is_transparent;
private:
template <typename _Tp, typename _Up>
static constexpr decltype(auto)
_S_cmp(_Tp&& __t, _Up&& __u, false_type)
{ return std::forward<_Tp>(__t) > std::forward<_Up>(__u); }
template <typename _Tp, typename _Up>
static constexpr bool
_S_cmp(_Tp&& __t, _Up&& __u, true_type) noexcept
{
return greater<const volatile void*>{}(
static_cast<const volatile void*>(std::forward<_Tp>(__t)),
static_cast<const volatile void*>(std::forward<_Up>(__u)));
}
template<typename _Tp, typename _Up, typename = void>
struct __not_overloaded2 : true_type { };
template<typename _Tp, typename _Up>
struct __not_overloaded2<_Tp, _Up, __void_t<
decltype(std::declval<_Tp>().operator>(std::declval<_Up>()))>>
: false_type { };
template<typename _Tp, typename _Up, typename = void>
struct __not_overloaded : __not_overloaded2<_Tp, _Up> { };
template<typename _Tp, typename _Up>
struct __not_overloaded<_Tp, _Up, __void_t<
decltype(operator>(std::declval<_Tp>(), std::declval<_Up>()))>>
: false_type { };
template<typename _Tp, typename _Up>
using __ptr_cmp = __and_<__not_overloaded<_Tp, _Up>,
is_convertible<_Tp, const volatile void*>,
is_convertible<_Up, const volatile void*>>;
};
template<>
struct less<void>
{
template <typename _Tp, typename _Up>
constexpr auto
operator()(_Tp&& __t, _Up&& __u) const
noexcept(noexcept(std::forward<_Tp>(__t) < std::forward<_Up>(__u)))
-> decltype(std::forward<_Tp>(__t) < std::forward<_Up>(__u))
{
return _S_cmp(std::forward<_Tp>(__t), std::forward<_Up>(__u),
__ptr_cmp<_Tp, _Up>{});
}
template<typename _Tp, typename _Up>
constexpr bool
operator()(_Tp* __t, _Up* __u) const noexcept
{ return less<common_type_t<_Tp*, _Up*>>{}(__t, __u); }
typedef __is_transparent is_transparent;
private:
template <typename _Tp, typename _Up>
static constexpr decltype(auto)
_S_cmp(_Tp&& __t, _Up&& __u, false_type)
{ return std::forward<_Tp>(__t) < std::forward<_Up>(__u); }
template <typename _Tp, typename _Up>
static constexpr bool
_S_cmp(_Tp&& __t, _Up&& __u, true_type) noexcept
{
return less<const volatile void*>{}(
static_cast<const volatile void*>(std::forward<_Tp>(__t)),
static_cast<const volatile void*>(std::forward<_Up>(__u)));
}
template<typename _Tp, typename _Up, typename = void>
struct __not_overloaded2 : true_type { };
template<typename _Tp, typename _Up>
struct __not_overloaded2<_Tp, _Up, __void_t<
decltype(std::declval<_Tp>().operator<(std::declval<_Up>()))>>
: false_type { };
template<typename _Tp, typename _Up, typename = void>
struct __not_overloaded : __not_overloaded2<_Tp, _Up> { };
template<typename _Tp, typename _Up>
struct __not_overloaded<_Tp, _Up, __void_t<
decltype(operator<(std::declval<_Tp>(), std::declval<_Up>()))>>
: false_type { };
template<typename _Tp, typename _Up>
using __ptr_cmp = __and_<__not_overloaded<_Tp, _Up>,
is_convertible<_Tp, const volatile void*>,
is_convertible<_Up, const volatile void*>>;
};
template<>
struct greater_equal<void>
{
template <typename _Tp, typename _Up>
constexpr auto
operator()(_Tp&& __t, _Up&& __u) const
noexcept(noexcept(std::forward<_Tp>(__t) >= std::forward<_Up>(__u)))
-> decltype(std::forward<_Tp>(__t) >= std::forward<_Up>(__u))
{
return _S_cmp(std::forward<_Tp>(__t), std::forward<_Up>(__u),
__ptr_cmp<_Tp, _Up>{});
}
template<typename _Tp, typename _Up>
constexpr bool
operator()(_Tp* __t, _Up* __u) const noexcept
{ return greater_equal<common_type_t<_Tp*, _Up*>>{}(__t, __u); }
typedef __is_transparent is_transparent;
private:
template <typename _Tp, typename _Up>
static constexpr decltype(auto)
_S_cmp(_Tp&& __t, _Up&& __u, false_type)
{ return std::forward<_Tp>(__t) >= std::forward<_Up>(__u); }
template <typename _Tp, typename _Up>
static constexpr bool
_S_cmp(_Tp&& __t, _Up&& __u, true_type) noexcept
{
return greater_equal<const volatile void*>{}(
static_cast<const volatile void*>(std::forward<_Tp>(__t)),
static_cast<const volatile void*>(std::forward<_Up>(__u)));
}
template<typename _Tp, typename _Up, typename = void>
struct __not_overloaded2 : true_type { };
template<typename _Tp, typename _Up>
struct __not_overloaded2<_Tp, _Up, __void_t<
decltype(std::declval<_Tp>().operator>=(std::declval<_Up>()))>>
: false_type { };
template<typename _Tp, typename _Up, typename = void>
struct __not_overloaded : __not_overloaded2<_Tp, _Up> { };
template<typename _Tp, typename _Up>
struct __not_overloaded<_Tp, _Up, __void_t<
decltype(operator>=(std::declval<_Tp>(), std::declval<_Up>()))>>
: false_type { };
template<typename _Tp, typename _Up>
using __ptr_cmp = __and_<__not_overloaded<_Tp, _Up>,
is_convertible<_Tp, const volatile void*>,
is_convertible<_Up, const volatile void*>>;
};
template<>
struct less_equal<void>
{
template <typename _Tp, typename _Up>
constexpr auto
operator()(_Tp&& __t, _Up&& __u) const
noexcept(noexcept(std::forward<_Tp>(__t) <= std::forward<_Up>(__u)))
-> decltype(std::forward<_Tp>(__t) <= std::forward<_Up>(__u))
{
return _S_cmp(std::forward<_Tp>(__t), std::forward<_Up>(__u),
__ptr_cmp<_Tp, _Up>{});
}
template<typename _Tp, typename _Up>
constexpr bool
operator()(_Tp* __t, _Up* __u) const noexcept
{ return less_equal<common_type_t<_Tp*, _Up*>>{}(__t, __u); }
typedef __is_transparent is_transparent;
private:
template <typename _Tp, typename _Up>
static constexpr decltype(auto)
_S_cmp(_Tp&& __t, _Up&& __u, false_type)
{ return std::forward<_Tp>(__t) <= std::forward<_Up>(__u); }
template <typename _Tp, typename _Up>
static constexpr bool
_S_cmp(_Tp&& __t, _Up&& __u, true_type) noexcept
{
return less_equal<const volatile void*>{}(
static_cast<const volatile void*>(std::forward<_Tp>(__t)),
static_cast<const volatile void*>(std::forward<_Up>(__u)));
}
template<typename _Tp, typename _Up, typename = void>
struct __not_overloaded2 : true_type { };
template<typename _Tp, typename _Up>
struct __not_overloaded2<_Tp, _Up, __void_t<
decltype(std::declval<_Tp>().operator<=(std::declval<_Up>()))>>
: false_type { };
template<typename _Tp, typename _Up, typename = void>
struct __not_overloaded : __not_overloaded2<_Tp, _Up> { };
template<typename _Tp, typename _Up>
struct __not_overloaded<_Tp, _Up, __void_t<
decltype(operator<=(std::declval<_Tp>(), std::declval<_Up>()))>>
: false_type { };
template<typename _Tp, typename _Up>
using __ptr_cmp = __and_<__not_overloaded<_Tp, _Up>,
is_convertible<_Tp, const volatile void*>,
is_convertible<_Up, const volatile void*>>;
};
template<typename _Tp = void>
struct logical_and;
template<typename _Tp = void>
struct logical_or;
template<typename _Tp = void>
struct logical_not;
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Tp>
struct logical_and : public binary_function<_Tp, _Tp, bool>
{
constexpr
bool
operator()(const _Tp& __x, const _Tp& __y) const
{ return __x && __y; }
};
template<typename _Tp>
struct logical_or : public binary_function<_Tp, _Tp, bool>
{
constexpr
bool
operator()(const _Tp& __x, const _Tp& __y) const
{ return __x || __y; }
};
template<typename _Tp>
struct logical_not : public unary_function<_Tp, bool>
{
constexpr
bool
operator()(const _Tp& __x) const
{ return !__x; }
};
#pragma GCC diagnostic pop
template<>
struct logical_and<void>
{
template <typename _Tp, typename _Up>
constexpr
auto
operator()(_Tp&& __t, _Up&& __u) const
noexcept(noexcept(std::forward<_Tp>(__t) && std::forward<_Up>(__u)))
-> decltype(std::forward<_Tp>(__t) && std::forward<_Up>(__u))
{ return std::forward<_Tp>(__t) && std::forward<_Up>(__u); }
typedef __is_transparent is_transparent;
};
template<>
struct logical_or<void>
{
template <typename _Tp, typename _Up>
constexpr
auto
operator()(_Tp&& __t, _Up&& __u) const
noexcept(noexcept(std::forward<_Tp>(__t) || std::forward<_Up>(__u)))
-> decltype(std::forward<_Tp>(__t) || std::forward<_Up>(__u))
{ return std::forward<_Tp>(__t) || std::forward<_Up>(__u); }
typedef __is_transparent is_transparent;
};
template<>
struct logical_not<void>
{
template <typename _Tp>
constexpr
auto
operator()(_Tp&& __t) const
noexcept(noexcept(!std::forward<_Tp>(__t)))
-> decltype(!std::forward<_Tp>(__t))
{ return !std::forward<_Tp>(__t); }
typedef __is_transparent is_transparent;
};
template<typename _Tp = void>
struct bit_and;
template<typename _Tp = void>
struct bit_or;
template<typename _Tp = void>
struct bit_xor;
template<typename _Tp = void>
struct bit_not;
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Tp>
struct bit_and : public binary_function<_Tp, _Tp, _Tp>
{
constexpr
_Tp
operator()(const _Tp& __x, const _Tp& __y) const
{ return __x & __y; }
};
template<typename _Tp>
struct bit_or : public binary_function<_Tp, _Tp, _Tp>
{
constexpr
_Tp
operator()(const _Tp& __x, const _Tp& __y) const
{ return __x | __y; }
};
template<typename _Tp>
struct bit_xor : public binary_function<_Tp, _Tp, _Tp>
{
constexpr
_Tp
operator()(const _Tp& __x, const _Tp& __y) const
{ return __x ^ __y; }
};
template<typename _Tp>
struct bit_not : public unary_function<_Tp, _Tp>
{
constexpr
_Tp
operator()(const _Tp& __x) const
{ return ~__x; }
};
#pragma GCC diagnostic pop
template <>
struct bit_and<void>
{
template <typename _Tp, typename _Up>
constexpr
auto
operator()(_Tp&& __t, _Up&& __u) const
noexcept(noexcept(std::forward<_Tp>(__t) & std::forward<_Up>(__u)))
-> decltype(std::forward<_Tp>(__t) & std::forward<_Up>(__u))
{ return std::forward<_Tp>(__t) & std::forward<_Up>(__u); }
typedef __is_transparent is_transparent;
};
template <>
struct bit_or<void>
{
template <typename _Tp, typename _Up>
constexpr
auto
operator()(_Tp&& __t, _Up&& __u) const
noexcept(noexcept(std::forward<_Tp>(__t) | std::forward<_Up>(__u)))
-> decltype(std::forward<_Tp>(__t) | std::forward<_Up>(__u))
{ return std::forward<_Tp>(__t) | std::forward<_Up>(__u); }
typedef __is_transparent is_transparent;
};
template <>
struct bit_xor<void>
{
template <typename _Tp, typename _Up>
constexpr
auto
operator()(_Tp&& __t, _Up&& __u) const
noexcept(noexcept(std::forward<_Tp>(__t) ^ std::forward<_Up>(__u)))
-> decltype(std::forward<_Tp>(__t) ^ std::forward<_Up>(__u))
{ return std::forward<_Tp>(__t) ^ std::forward<_Up>(__u); }
typedef __is_transparent is_transparent;
};
template <>
struct bit_not<void>
{
template <typename _Tp>
constexpr
auto
operator()(_Tp&& __t) const
noexcept(noexcept(~std::forward<_Tp>(__t)))
-> decltype(~std::forward<_Tp>(__t))
{ return ~std::forward<_Tp>(__t); }
typedef __is_transparent is_transparent;
};
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Predicate>
class [[__deprecated__]] unary_negate
: public unary_function<typename _Predicate::argument_type, bool>
{
protected:
_Predicate _M_pred;
public:
constexpr
explicit
unary_negate(const _Predicate& __x) : _M_pred(__x) { }
constexpr
bool
operator()(const typename _Predicate::argument_type& __x) const
{ return !_M_pred(__x); }
};
template<typename _Predicate>
__attribute__ ((__deprecated__ ("use '" "std::not_fn" "' instead")))
constexpr
inline unary_negate<_Predicate>
not1(const _Predicate& __pred)
{ return unary_negate<_Predicate>(__pred); }
template<typename _Predicate>
class [[__deprecated__]] binary_negate
: public binary_function<typename _Predicate::first_argument_type,
typename _Predicate::second_argument_type, bool>
{
protected:
_Predicate _M_pred;
public:
constexpr
explicit
binary_negate(const _Predicate& __x) : _M_pred(__x) { }
constexpr
bool
operator()(const typename _Predicate::first_argument_type& __x,
const typename _Predicate::second_argument_type& __y) const
{ return !_M_pred(__x, __y); }
};
template<typename _Predicate>
__attribute__ ((__deprecated__ ("use '" "std::not_fn" "' instead")))
constexpr
inline binary_negate<_Predicate>
not2(const _Predicate& __pred)
{ return binary_negate<_Predicate>(__pred); }
template<typename _Arg, typename _Result>
class pointer_to_unary_function : public unary_function<_Arg, _Result>
{
protected:
_Result (*_M_ptr)(_Arg);
public:
pointer_to_unary_function() { }
explicit
pointer_to_unary_function(_Result (*__x)(_Arg))
: _M_ptr(__x) { }
_Result
operator()(_Arg __x) const
{ return _M_ptr(__x); }
} __attribute__ ((__deprecated__));
template<typename _Arg, typename _Result>
__attribute__ ((__deprecated__ ("use '" "std::function" "' instead")))
inline pointer_to_unary_function<_Arg, _Result>
ptr_fun(_Result (*__x)(_Arg))
{ return pointer_to_unary_function<_Arg, _Result>(__x); }
template<typename _Arg1, typename _Arg2, typename _Result>
class pointer_to_binary_function
: public binary_function<_Arg1, _Arg2, _Result>
{
protected:
_Result (*_M_ptr)(_Arg1, _Arg2);
public:
pointer_to_binary_function() { }
explicit
pointer_to_binary_function(_Result (*__x)(_Arg1, _Arg2))
: _M_ptr(__x) { }
_Result
operator()(_Arg1 __x, _Arg2 __y) const
{ return _M_ptr(__x, __y); }
} __attribute__ ((__deprecated__));
template<typename _Arg1, typename _Arg2, typename _Result>
__attribute__ ((__deprecated__ ("use '" "std::function" "' instead")))
inline pointer_to_binary_function<_Arg1, _Arg2, _Result>
ptr_fun(_Result (*__x)(_Arg1, _Arg2))
{ return pointer_to_binary_function<_Arg1, _Arg2, _Result>(__x); }
template<typename _Tp>
struct _Identity
: public unary_function<_Tp, _Tp>
{
_Tp&
operator()(_Tp& __x) const
{ return __x; }
const _Tp&
operator()(const _Tp& __x) const
{ return __x; }
};
template<typename _Tp> struct _Identity<const _Tp> : _Identity<_Tp> { };
template<typename _Pair>
struct _Select1st
: public unary_function<_Pair, typename _Pair::first_type>
{
typename _Pair::first_type&
operator()(_Pair& __x) const
{ return __x.first; }
const typename _Pair::first_type&
operator()(const _Pair& __x) const
{ return __x.first; }
template<typename _Pair2>
typename _Pair2::first_type&
operator()(_Pair2& __x) const
{ return __x.first; }
template<typename _Pair2>
const typename _Pair2::first_type&
operator()(const _Pair2& __x) const
{ return __x.first; }
};
template<typename _Pair>
struct _Select2nd
: public unary_function<_Pair, typename _Pair::second_type>
{
typename _Pair::second_type&
operator()(_Pair& __x) const
{ return __x.second; }
const typename _Pair::second_type&
operator()(const _Pair& __x) const
{ return __x.second; }
};
template<typename _Ret, typename _Tp>
class mem_fun_t : public unary_function<_Tp*, _Ret>
{
public:
explicit
mem_fun_t(_Ret (_Tp::*__pf)())
: _M_f(__pf) { }
_Ret
operator()(_Tp* __p) const
{ return (__p->*_M_f)(); }
private:
_Ret (_Tp::*_M_f)();
} __attribute__ ((__deprecated__));
template<typename _Ret, typename _Tp>
class const_mem_fun_t : public unary_function<const _Tp*, _Ret>
{
public:
explicit
const_mem_fun_t(_Ret (_Tp::*__pf)() const)
: _M_f(__pf) { }
_Ret
operator()(const _Tp* __p) const
{ return (__p->*_M_f)(); }
private:
_Ret (_Tp::*_M_f)() const;
} __attribute__ ((__deprecated__));
template<typename _Ret, typename _Tp>
class mem_fun_ref_t : public unary_function<_Tp, _Ret>
{
public:
explicit
mem_fun_ref_t(_Ret (_Tp::*__pf)())
: _M_f(__pf) { }
_Ret
operator()(_Tp& __r) const
{ return (__r.*_M_f)(); }
private:
_Ret (_Tp::*_M_f)();
} __attribute__ ((__deprecated__));
template<typename _Ret, typename _Tp>
class const_mem_fun_ref_t : public unary_function<_Tp, _Ret>
{
public:
explicit
const_mem_fun_ref_t(_Ret (_Tp::*__pf)() const)
: _M_f(__pf) { }
_Ret
operator()(const _Tp& __r) const
{ return (__r.*_M_f)(); }
private:
_Ret (_Tp::*_M_f)() const;
} __attribute__ ((__deprecated__));
template<typename _Ret, typename _Tp, typename _Arg>
class mem_fun1_t : public binary_function<_Tp*, _Arg, _Ret>
{
public:
explicit
mem_fun1_t(_Ret (_Tp::*__pf)(_Arg))
: _M_f(__pf) { }
_Ret
operator()(_Tp* __p, _Arg __x) const
{ return (__p->*_M_f)(__x); }
private:
_Ret (_Tp::*_M_f)(_Arg);
} __attribute__ ((__deprecated__));
template<typename _Ret, typename _Tp, typename _Arg>
class const_mem_fun1_t : public binary_function<const _Tp*, _Arg, _Ret>
{
public:
explicit
const_mem_fun1_t(_Ret (_Tp::*__pf)(_Arg) const)
: _M_f(__pf) { }
_Ret
operator()(const _Tp* __p, _Arg __x) const
{ return (__p->*_M_f)(__x); }
private:
_Ret (_Tp::*_M_f)(_Arg) const;
} __attribute__ ((__deprecated__));
template<typename _Ret, typename _Tp, typename _Arg>
class mem_fun1_ref_t : public binary_function<_Tp, _Arg, _Ret>
{
public:
explicit
mem_fun1_ref_t(_Ret (_Tp::*__pf)(_Arg))
: _M_f(__pf) { }
_Ret
operator()(_Tp& __r, _Arg __x) const
{ return (__r.*_M_f)(__x); }
private:
_Ret (_Tp::*_M_f)(_Arg);
} __attribute__ ((__deprecated__));
template<typename _Ret, typename _Tp, typename _Arg>
class const_mem_fun1_ref_t : public binary_function<_Tp, _Arg, _Ret>
{
public:
explicit
const_mem_fun1_ref_t(_Ret (_Tp::*__pf)(_Arg) const)
: _M_f(__pf) { }
_Ret
operator()(const _Tp& __r, _Arg __x) const
{ return (__r.*_M_f)(__x); }
private:
_Ret (_Tp::*_M_f)(_Arg) const;
} __attribute__ ((__deprecated__));
template<typename _Ret, typename _Tp>
__attribute__ ((__deprecated__ ("use '" "std::mem_fn" "' instead")))
inline mem_fun_t<_Ret, _Tp>
mem_fun(_Ret (_Tp::*__f)())
{ return mem_fun_t<_Ret, _Tp>(__f); }
template<typename _Ret, typename _Tp>
__attribute__ ((__deprecated__ ("use '" "std::mem_fn" "' instead")))
inline const_mem_fun_t<_Ret, _Tp>
mem_fun(_Ret (_Tp::*__f)() const)
{ return const_mem_fun_t<_Ret, _Tp>(__f); }
template<typename _Ret, typename _Tp>
__attribute__ ((__deprecated__ ("use '" "std::mem_fn" "' instead")))
inline mem_fun_ref_t<_Ret, _Tp>
mem_fun_ref(_Ret (_Tp::*__f)())
{ return mem_fun_ref_t<_Ret, _Tp>(__f); }
template<typename _Ret, typename _Tp>
__attribute__ ((__deprecated__ ("use '" "std::mem_fn" "' instead")))
inline const_mem_fun_ref_t<_Ret, _Tp>
mem_fun_ref(_Ret (_Tp::*__f)() const)
{ return const_mem_fun_ref_t<_Ret, _Tp>(__f); }
template<typename _Ret, typename _Tp, typename _Arg>
__attribute__ ((__deprecated__ ("use '" "std::mem_fn" "' instead")))
inline mem_fun1_t<_Ret, _Tp, _Arg>
mem_fun(_Ret (_Tp::*__f)(_Arg))
{ return mem_fun1_t<_Ret, _Tp, _Arg>(__f); }
template<typename _Ret, typename _Tp, typename _Arg>
__attribute__ ((__deprecated__ ("use '" "std::mem_fn" "' instead")))
inline const_mem_fun1_t<_Ret, _Tp, _Arg>
mem_fun(_Ret (_Tp::*__f)(_Arg) const)
{ return const_mem_fun1_t<_Ret, _Tp, _Arg>(__f); }
template<typename _Ret, typename _Tp, typename _Arg>
__attribute__ ((__deprecated__ ("use '" "std::mem_fn" "' instead")))
inline mem_fun1_ref_t<_Ret, _Tp, _Arg>
mem_fun_ref(_Ret (_Tp::*__f)(_Arg))
{ return mem_fun1_ref_t<_Ret, _Tp, _Arg>(__f); }
template<typename _Ret, typename _Tp, typename _Arg>
__attribute__ ((__deprecated__ ("use '" "std::mem_fn" "' instead")))
inline const_mem_fun1_ref_t<_Ret, _Tp, _Arg>
mem_fun_ref(_Ret (_Tp::*__f)(_Arg) const)
{ return const_mem_fun1_ref_t<_Ret, _Tp, _Arg>(__f); }
#pragma GCC diagnostic pop
template<typename _Func, typename _SfinaeType, typename = __void_t<>>
struct __has_is_transparent
{ };
template<typename _Func, typename _SfinaeType>
struct __has_is_transparent<_Func, _SfinaeType,
__void_t<typename _Func::is_transparent>>
{ typedef void type; };
template<typename _Func, typename _SfinaeType>
using __has_is_transparent_t
= typename __has_is_transparent<_Func, _SfinaeType>::type;
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Operation>
class binder1st
: public unary_function<typename _Operation::second_argument_type,
typename _Operation::result_type>
{
protected:
_Operation op;
typename _Operation::first_argument_type value;
public:
binder1st(const _Operation& __x,
const typename _Operation::first_argument_type& __y)
: op(__x), value(__y) { }
typename _Operation::result_type
operator()(const typename _Operation::second_argument_type& __x) const
{ return op(value, __x); }
typename _Operation::result_type
operator()(typename _Operation::second_argument_type& __x) const
{ return op(value, __x); }
} __attribute__ ((__deprecated__ ("use '" "std::bind" "' instead")));
template<typename _Operation, typename _Tp>
__attribute__ ((__deprecated__ ("use '" "std::bind" "' instead")))
inline binder1st<_Operation>
bind1st(const _Operation& __fn, const _Tp& __x)
{
typedef typename _Operation::first_argument_type _Arg1_type;
return binder1st<_Operation>(__fn, _Arg1_type(__x));
}
template<typename _Operation>
class binder2nd
: public unary_function<typename _Operation::first_argument_type,
typename _Operation::result_type>
{
protected:
_Operation op;
typename _Operation::second_argument_type value;
public:
binder2nd(const _Operation& __x,
const typename _Operation::second_argument_type& __y)
: op(__x), value(__y) { }
typename _Operation::result_type
operator()(const typename _Operation::first_argument_type& __x) const
{ return op(__x, value); }
typename _Operation::result_type
operator()(typename _Operation::first_argument_type& __x) const
{ return op(__x, value); }
} __attribute__ ((__deprecated__ ("use '" "std::bind" "' instead")));
template<typename _Operation, typename _Tp>
__attribute__ ((__deprecated__ ("use '" "std::bind" "' instead")))
inline binder2nd<_Operation>
bind2nd(const _Operation& __fn, const _Tp& __x)
{
typedef typename _Operation::second_argument_type _Arg2_type;
return binder2nd<_Operation>(__fn, _Arg2_type(__x));
}
}
#pragma GCC diagnostic pop
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Res, typename... _ArgTypes>
struct _Maybe_unary_or_binary_function { };
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Res, typename _T1>
struct _Maybe_unary_or_binary_function<_Res, _T1>
: std::unary_function<_T1, _Res> { };
template<typename _Res, typename _T1, typename _T2>
struct _Maybe_unary_or_binary_function<_Res, _T1, _T2>
: std::binary_function<_T1, _T2, _Res> { };
#pragma GCC diagnostic pop
template<typename _Signature>
struct _Mem_fn_traits;
template<typename _Res, typename _Class, typename... _ArgTypes>
struct _Mem_fn_traits_base
{
using __result_type = _Res;
using __maybe_type
= _Maybe_unary_or_binary_function<_Res, _Class*, _ArgTypes...>;
using __arity = integral_constant<size_t, sizeof...(_ArgTypes)>;
};
template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) > : _Mem_fn_traits_base<_Res, _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) > : _Mem_fn_traits_base<_Res, _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) const > : _Mem_fn_traits_base<_Res, const _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) const > : _Mem_fn_traits_base<_Res, const _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) volatile > : _Mem_fn_traits_base<_Res, volatile _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) volatile > : _Mem_fn_traits_base<_Res, volatile _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) const volatile > : _Mem_fn_traits_base<_Res, const volatile _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) const volatile > : _Mem_fn_traits_base<_Res, const volatile _Class, _ArgTypes...> { using __vararg = true_type; };
template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) &> : _Mem_fn_traits_base<_Res, _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) &> : _Mem_fn_traits_base<_Res, _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) const &> : _Mem_fn_traits_base<_Res, const _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) const &> : _Mem_fn_traits_base<_Res, const _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) volatile &> : _Mem_fn_traits_base<_Res, volatile _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) volatile &> : _Mem_fn_traits_base<_Res, volatile _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) const volatile &> : _Mem_fn_traits_base<_Res, const volatile _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) const volatile &> : _Mem_fn_traits_base<_Res, const volatile _Class, _ArgTypes...> { using __vararg = true_type; };
template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) &&> : _Mem_fn_traits_base<_Res, _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) &&> : _Mem_fn_traits_base<_Res, _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) const &&> : _Mem_fn_traits_base<_Res, const _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) const &&> : _Mem_fn_traits_base<_Res, const _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) volatile &&> : _Mem_fn_traits_base<_Res, volatile _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) volatile &&> : _Mem_fn_traits_base<_Res, volatile _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) const volatile &&> : _Mem_fn_traits_base<_Res, const volatile _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) const volatile &&> : _Mem_fn_traits_base<_Res, const volatile _Class, _ArgTypes...> { using __vararg = true_type; };
template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) noexcept> : _Mem_fn_traits_base<_Res, _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) noexcept> : _Mem_fn_traits_base<_Res, _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) const noexcept> : _Mem_fn_traits_base<_Res, const _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) const noexcept> : _Mem_fn_traits_base<_Res, const _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) volatile noexcept> : _Mem_fn_traits_base<_Res, volatile _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) volatile noexcept> : _Mem_fn_traits_base<_Res, volatile _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) const volatile noexcept> : _Mem_fn_traits_base<_Res, const volatile _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) const volatile noexcept> : _Mem_fn_traits_base<_Res, const volatile _Class, _ArgTypes...> { using __vararg = true_type; };
template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) & noexcept> : _Mem_fn_traits_base<_Res, _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) & noexcept> : _Mem_fn_traits_base<_Res, _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) const & noexcept> : _Mem_fn_traits_base<_Res, const _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) const & noexcept> : _Mem_fn_traits_base<_Res, const _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) volatile & noexcept> : _Mem_fn_traits_base<_Res, volatile _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) volatile & noexcept> : _Mem_fn_traits_base<_Res, volatile _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) const volatile & noexcept> : _Mem_fn_traits_base<_Res, const volatile _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) const volatile & noexcept> : _Mem_fn_traits_base<_Res, const volatile _Class, _ArgTypes...> { using __vararg = true_type; };
template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) && noexcept> : _Mem_fn_traits_base<_Res, _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) && noexcept> : _Mem_fn_traits_base<_Res, _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) const && noexcept> : _Mem_fn_traits_base<_Res, const _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) const && noexcept> : _Mem_fn_traits_base<_Res, const _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) volatile && noexcept> : _Mem_fn_traits_base<_Res, volatile _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) volatile && noexcept> : _Mem_fn_traits_base<_Res, volatile _Class, _ArgTypes...> { using __vararg = true_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes...) const volatile && noexcept> : _Mem_fn_traits_base<_Res, const volatile _Class, _ArgTypes...> { using __vararg = false_type; }; template<typename _Res, typename _Class, typename... _ArgTypes> struct _Mem_fn_traits<_Res (_Class::*)(_ArgTypes... ...) const volatile && noexcept> : _Mem_fn_traits_base<_Res, const volatile _Class, _ArgTypes...> { using __vararg = true_type; };
template<typename _Functor, typename = __void_t<>>
struct _Maybe_get_result_type
{ };
template<typename _Functor>
struct _Maybe_get_result_type<_Functor,
__void_t<typename _Functor::result_type>>
{ typedef typename _Functor::result_type result_type; };
template<typename _Functor>
struct _Weak_result_type_impl
: _Maybe_get_result_type<_Functor>
{ };
template<typename _Res, typename... _ArgTypes , bool _NE>
struct _Weak_result_type_impl<_Res(_ArgTypes...) noexcept (_NE)>
{ typedef _Res result_type; };
template<typename _Res, typename... _ArgTypes , bool _NE>
struct _Weak_result_type_impl<_Res(_ArgTypes......) noexcept (_NE)>
{ typedef _Res result_type; };
template<typename _Res, typename... _ArgTypes , bool _NE>
struct _Weak_result_type_impl<_Res(*)(_ArgTypes...) noexcept (_NE)>
{ typedef _Res result_type; };
template<typename _Res, typename... _ArgTypes , bool _NE>
struct
_Weak_result_type_impl<_Res(*)(_ArgTypes......) noexcept (_NE)>
{ typedef _Res result_type; };
template<typename _Functor,
bool = is_member_function_pointer<_Functor>::value>
struct _Weak_result_type_memfun
: _Weak_result_type_impl<_Functor>
{ };
template<typename _MemFunPtr>
struct _Weak_result_type_memfun<_MemFunPtr, true>
{
using result_type = typename _Mem_fn_traits<_MemFunPtr>::__result_type;
};
template<typename _Func, typename _Class>
struct _Weak_result_type_memfun<_Func _Class::*, false>
{ };
template<typename _Functor>
struct _Weak_result_type
: _Weak_result_type_memfun<typename remove_cv<_Functor>::type>
{ };
template<typename _Tp, typename = __void_t<>>
struct _Refwrap_base_arg1
{ };
template<typename _Tp>
struct _Refwrap_base_arg1<_Tp,
__void_t<typename _Tp::argument_type>>
{
typedef typename _Tp::argument_type argument_type;
};
template<typename _Tp, typename = __void_t<>>
struct _Refwrap_base_arg2
{ };
template<typename _Tp>
struct _Refwrap_base_arg2<_Tp,
__void_t<typename _Tp::first_argument_type,
typename _Tp::second_argument_type>>
{
typedef typename _Tp::first_argument_type first_argument_type;
typedef typename _Tp::second_argument_type second_argument_type;
};
template<typename _Tp>
struct _Reference_wrapper_base
: _Weak_result_type<_Tp>, _Refwrap_base_arg1<_Tp>, _Refwrap_base_arg2<_Tp>
{ };
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Res, typename _T1 , bool _NE>
struct _Reference_wrapper_base<_Res(_T1) noexcept (_NE)>
: unary_function<_T1, _Res>
{ };
template<typename _Res, typename _T1>
struct _Reference_wrapper_base<_Res(_T1) const>
: unary_function<_T1, _Res>
{ };
template<typename _Res, typename _T1>
struct _Reference_wrapper_base<_Res(_T1) volatile>
: unary_function<_T1, _Res>
{ };
template<typename _Res, typename _T1>
struct _Reference_wrapper_base<_Res(_T1) const volatile>
: unary_function<_T1, _Res>
{ };
template<typename _Res, typename _T1, typename _T2 , bool _NE>
struct _Reference_wrapper_base<_Res(_T1, _T2) noexcept (_NE)>
: binary_function<_T1, _T2, _Res>
{ };
template<typename _Res, typename _T1, typename _T2>
struct _Reference_wrapper_base<_Res(_T1, _T2) const>
: binary_function<_T1, _T2, _Res>
{ };
template<typename _Res, typename _T1, typename _T2>
struct _Reference_wrapper_base<_Res(_T1, _T2) volatile>
: binary_function<_T1, _T2, _Res>
{ };
template<typename _Res, typename _T1, typename _T2>
struct _Reference_wrapper_base<_Res(_T1, _T2) const volatile>
: binary_function<_T1, _T2, _Res>
{ };
template<typename _Res, typename _T1 , bool _NE>
struct _Reference_wrapper_base<_Res(*)(_T1) noexcept (_NE)>
: unary_function<_T1, _Res>
{ };
template<typename _Res, typename _T1, typename _T2 , bool _NE>
struct _Reference_wrapper_base<_Res(*)(_T1, _T2) noexcept (_NE)>
: binary_function<_T1, _T2, _Res>
{ };
template<typename _Tp, bool = is_member_function_pointer<_Tp>::value>
struct _Reference_wrapper_base_memfun
: _Reference_wrapper_base<_Tp>
{ };
template<typename _MemFunPtr>
struct _Reference_wrapper_base_memfun<_MemFunPtr, true>
: _Mem_fn_traits<_MemFunPtr>::__maybe_type
{
using result_type = typename _Mem_fn_traits<_MemFunPtr>::__result_type;
};
#pragma GCC diagnostic pop
template<typename _Tp>
class reference_wrapper
: public _Reference_wrapper_base_memfun<typename remove_cv<_Tp>::type>
{
_Tp* _M_data;
static _Tp* _S_fun(_Tp& __r) noexcept { return std::__addressof(__r); }
static void _S_fun(_Tp&&) = delete;
template<typename _Up, typename _Up2 = __remove_cvref_t<_Up>>
using __not_same
= typename enable_if<!is_same<reference_wrapper, _Up2>::value>::type;
public:
typedef _Tp type;
template<typename _Up, typename = __not_same<_Up>, typename
= decltype(reference_wrapper::_S_fun(std::declval<_Up>()))>
reference_wrapper(_Up&& __uref)
noexcept(noexcept(reference_wrapper::_S_fun(std::declval<_Up>())))
: _M_data(reference_wrapper::_S_fun(std::forward<_Up>(__uref)))
{ }
reference_wrapper(const reference_wrapper&) = default;
reference_wrapper&
operator=(const reference_wrapper&) = default;
operator _Tp&() const noexcept
{ return this->get(); }
_Tp&
get() const noexcept
{ return *_M_data; }
template<typename... _Args>
typename __invoke_result<_Tp&, _Args...>::type
operator()(_Args&&... __args) const
noexcept(__is_nothrow_invocable<_Tp&, _Args...>::value)
{
return std::__invoke(get(), std::forward<_Args>(__args)...);
}
};
template<typename _Tp>
reference_wrapper(_Tp&) -> reference_wrapper<_Tp>;
template<typename _Tp>
inline reference_wrapper<_Tp>
ref(_Tp& __t) noexcept
{ return reference_wrapper<_Tp>(__t); }
template<typename _Tp>
inline reference_wrapper<const _Tp>
cref(const _Tp& __t) noexcept
{ return reference_wrapper<const _Tp>(__t); }
template<typename _Tp>
void ref(const _Tp&&) = delete;
template<typename _Tp>
void cref(const _Tp&&) = delete;
template<typename _Tp>
inline reference_wrapper<_Tp>
ref(reference_wrapper<_Tp> __t) noexcept
{ return __t; }
template<typename _Tp>
inline reference_wrapper<const _Tp>
cref(reference_wrapper<_Tp> __t) noexcept
{ return { __t.get() }; }
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Container>
[[__nodiscard__, __gnu__::__always_inline__]]
inline constexpr auto
begin(_Container& __cont) -> decltype(__cont.begin())
{ return __cont.begin(); }
template<typename _Container>
[[__nodiscard__, __gnu__::__always_inline__]]
inline constexpr auto
begin(const _Container& __cont) -> decltype(__cont.begin())
{ return __cont.begin(); }
template<typename _Container>
[[__nodiscard__, __gnu__::__always_inline__]]
inline constexpr auto
end(_Container& __cont) -> decltype(__cont.end())
{ return __cont.end(); }
template<typename _Container>
[[__nodiscard__, __gnu__::__always_inline__]]
inline constexpr auto
end(const _Container& __cont) -> decltype(__cont.end())
{ return __cont.end(); }
template<typename _Tp, size_t _Nm>
[[__nodiscard__, __gnu__::__always_inline__]]
inline constexpr _Tp*
begin(_Tp (&__arr)[_Nm]) noexcept
{ return __arr; }
template<typename _Tp, size_t _Nm>
[[__nodiscard__, __gnu__::__always_inline__]]
inline constexpr _Tp*
end(_Tp (&__arr)[_Nm]) noexcept
{ return __arr + _Nm; }
template<typename _Tp> class valarray;
template<typename _Tp> _Tp* begin(valarray<_Tp>&) noexcept;
template<typename _Tp> const _Tp* begin(const valarray<_Tp>&) noexcept;
template<typename _Tp> _Tp* end(valarray<_Tp>&) noexcept;
template<typename _Tp> const _Tp* end(const valarray<_Tp>&) noexcept;
template<typename _Container>
[[__nodiscard__, __gnu__::__always_inline__]]
constexpr auto
cbegin(const _Container& __cont) noexcept(noexcept(std::begin(__cont)))
-> decltype(std::begin(__cont))
{ return std::begin(__cont); }
template<typename _Container>
[[__nodiscard__, __gnu__::__always_inline__]]
constexpr auto
cend(const _Container& __cont) noexcept(noexcept(std::end(__cont)))
-> decltype(std::end(__cont))
{ return std::end(__cont); }
template<typename _Container>
[[__nodiscard__, __gnu__::__always_inline__]]
inline constexpr auto
rbegin(_Container& __cont) -> decltype(__cont.rbegin())
{ return __cont.rbegin(); }
template<typename _Container>
[[__nodiscard__, __gnu__::__always_inline__]]
inline constexpr auto
rbegin(const _Container& __cont) -> decltype(__cont.rbegin())
{ return __cont.rbegin(); }
template<typename _Container>
[[__nodiscard__, __gnu__::__always_inline__]]
inline constexpr auto
rend(_Container& __cont) -> decltype(__cont.rend())
{ return __cont.rend(); }
template<typename _Container>
[[__nodiscard__, __gnu__::__always_inline__]]
inline constexpr auto
rend(const _Container& __cont) -> decltype(__cont.rend())
{ return __cont.rend(); }
template<typename _Tp, size_t _Nm>
[[__nodiscard__]]
inline constexpr reverse_iterator<_Tp*>
rbegin(_Tp (&__arr)[_Nm]) noexcept
{ return reverse_iterator<_Tp*>(__arr + _Nm); }
template<typename _Tp, size_t _Nm>
[[__nodiscard__]]
inline constexpr reverse_iterator<_Tp*>
rend(_Tp (&__arr)[_Nm]) noexcept
{ return reverse_iterator<_Tp*>(__arr); }
template<typename _Tp>
[[__nodiscard__]]
inline constexpr reverse_iterator<const _Tp*>
rbegin(initializer_list<_Tp> __il) noexcept
{ return reverse_iterator<const _Tp*>(__il.end()); }
template<typename _Tp>
[[__nodiscard__]]
inline constexpr reverse_iterator<const _Tp*>
rend(initializer_list<_Tp> __il) noexcept
{ return reverse_iterator<const _Tp*>(__il.begin()); }
template<typename _Container>
[[__nodiscard__, __gnu__::__always_inline__]]
inline constexpr auto
crbegin(const _Container& __cont) -> decltype(std::rbegin(__cont))
{ return std::rbegin(__cont); }
template<typename _Container>
[[__nodiscard__, __gnu__::__always_inline__]]
inline constexpr auto
crend(const _Container& __cont) -> decltype(std::rend(__cont))
{ return std::rend(__cont); }
template <typename _Container>
[[nodiscard, __gnu__::__always_inline__]]
constexpr auto
size(const _Container& __cont) noexcept(noexcept(__cont.size()))
-> decltype(__cont.size())
{ return __cont.size(); }
template <typename _Tp, size_t _Nm>
[[nodiscard, __gnu__::__always_inline__]]
constexpr size_t
size(const _Tp (&)[_Nm]) noexcept
{ return _Nm; }
template <typename _Container>
[[nodiscard, __gnu__::__always_inline__]]
constexpr auto
empty(const _Container& __cont) noexcept(noexcept(__cont.empty()))
-> decltype(__cont.empty())
{ return __cont.empty(); }
template <typename _Tp, size_t _Nm>
[[nodiscard, __gnu__::__always_inline__]]
constexpr bool
empty(const _Tp (&)[_Nm]) noexcept
{ return false; }
template <typename _Tp>
[[nodiscard, __gnu__::__always_inline__]]
constexpr bool
empty(initializer_list<_Tp> __il) noexcept
{ return __il.size() == 0;}
template <typename _Container>
[[nodiscard, __gnu__::__always_inline__]]
constexpr auto
data(_Container& __cont) noexcept(noexcept(__cont.data()))
-> decltype(__cont.data())
{ return __cont.data(); }
template <typename _Container>
[[nodiscard, __gnu__::__always_inline__]]
constexpr auto
data(const _Container& __cont) noexcept(noexcept(__cont.data()))
-> decltype(__cont.data())
{ return __cont.data(); }
template <typename _Tp, size_t _Nm>
[[nodiscard, __gnu__::__always_inline__]]
constexpr _Tp*
data(_Tp (&__array)[_Nm]) noexcept
{ return __array; }
template <typename _Tp>
[[nodiscard, __gnu__::__always_inline__]]
constexpr const _Tp*
data(initializer_list<_Tp> __il) noexcept
{ return __il.begin(); }
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Tp, typename _Alloc>
void
vector<_Tp, _Alloc>::
reserve(size_type __n)
{
if (__n > this->max_size())
__throw_length_error(("vector::reserve"));
if (this->capacity() < __n)
{
const size_type __old_size = size();
pointer __tmp;
if constexpr (_S_use_relocate())
{
__tmp = this->_M_allocate(__n);
_S_relocate(this->_M_impl._M_start, this->_M_impl._M_finish,
__tmp, _M_get_Tp_allocator());
}
else
{
__tmp = _M_allocate_and_copy(__n,
std::__make_move_if_noexcept_iterator(this->_M_impl._M_start),
std::__make_move_if_noexcept_iterator(this->_M_impl._M_finish));
std::_Destroy(this->_M_impl._M_start, this->_M_impl._M_finish,
_M_get_Tp_allocator());
}
;
_M_deallocate(this->_M_impl._M_start,
this->_M_impl._M_end_of_storage
- this->_M_impl._M_start);
this->_M_impl._M_start = __tmp;
this->_M_impl._M_finish = __tmp + __old_size;
this->_M_impl._M_end_of_storage = this->_M_impl._M_start + __n;
}
}
template<typename _Tp, typename _Alloc>
template<typename... _Args>
typename vector<_Tp, _Alloc>::reference
vector<_Tp, _Alloc>::
emplace_back(_Args&&... __args)
{
if (this->_M_impl._M_finish != this->_M_impl._M_end_of_storage)
{
;
_Alloc_traits::construct(this->_M_impl, this->_M_impl._M_finish,
std::forward<_Args>(__args)...);
++this->_M_impl._M_finish;
;
}
else
_M_realloc_insert(end(), std::forward<_Args>(__args)...);
return back();
}
template<typename _Tp, typename _Alloc>
typename vector<_Tp, _Alloc>::iterator
vector<_Tp, _Alloc>::
insert(const_iterator __position, const value_type& __x)
{
const size_type __n = __position - begin();
if (this->_M_impl._M_finish != this->_M_impl._M_end_of_storage)
{
do { if (std::__is_constant_evaluated() && !bool(__position != const_iterator())) __builtin_unreachable(); } while (false);
if (!(__position != const_iterator()))
__builtin_unreachable();
if (__position == end())
{
;
_Alloc_traits::construct(this->_M_impl, this->_M_impl._M_finish,
__x);
++this->_M_impl._M_finish;
;
}
else
{
const auto __pos = begin() + (__position - cbegin());
_Temporary_value __x_copy(this, __x);
_M_insert_aux(__pos, std::move(__x_copy._M_val()));
}
}
else
_M_realloc_insert(begin() + (__position - cbegin()), __x);
return iterator(this->_M_impl._M_start + __n);
}
template<typename _Tp, typename _Alloc>
typename vector<_Tp, _Alloc>::iterator
vector<_Tp, _Alloc>::
_M_erase(iterator __position)
{
if (__position + 1 != end())
std::move(__position + 1, end(), __position);
--this->_M_impl._M_finish;
_Alloc_traits::destroy(this->_M_impl, this->_M_impl._M_finish);
;
return __position;
}
template<typename _Tp, typename _Alloc>
typename vector<_Tp, _Alloc>::iterator
vector<_Tp, _Alloc>::
_M_erase(iterator __first, iterator __last)
{
if (__first != __last)
{
if (__last != end())
std::move(__last, end(), __first);
_M_erase_at_end(__first.base() + (end() - __last));
}
return __first;
}
template<typename _Tp, typename _Alloc>
vector<_Tp, _Alloc>&
vector<_Tp, _Alloc>::
operator=(const vector<_Tp, _Alloc>& __x)
{
if (std::__addressof(__x) != this)
{
;
if (_Alloc_traits::_S_propagate_on_copy_assign())
{
if (!_Alloc_traits::_S_always_equal()
&& _M_get_Tp_allocator() != __x._M_get_Tp_allocator())
{
this->clear();
_M_deallocate(this->_M_impl._M_start,
this->_M_impl._M_end_of_storage
- this->_M_impl._M_start);
this->_M_impl._M_start = nullptr;
this->_M_impl._M_finish = nullptr;
this->_M_impl._M_end_of_storage = nullptr;
}
std::__alloc_on_copy(_M_get_Tp_allocator(),
__x._M_get_Tp_allocator());
}
const size_type __xlen = __x.size();
if (__xlen > capacity())
{
pointer __tmp = _M_allocate_and_copy(__xlen, __x.begin(),
__x.end());
std::_Destroy(this->_M_impl._M_start, this->_M_impl._M_finish,
_M_get_Tp_allocator());
_M_deallocate(this->_M_impl._M_start,
this->_M_impl._M_end_of_storage
- this->_M_impl._M_start);
this->_M_impl._M_start = __tmp;
this->_M_impl._M_end_of_storage = this->_M_impl._M_start + __xlen;
}
else if (size() >= __xlen)
{
std::_Destroy(std::copy(__x.begin(), __x.end(), begin()),
end(), _M_get_Tp_allocator());
}
else
{
std::copy(__x._M_impl._M_start, __x._M_impl._M_start + size(),
this->_M_impl._M_start);
std::__uninitialized_copy_a(__x._M_impl._M_start + size(),
__x._M_impl._M_finish,
this->_M_impl._M_finish,
_M_get_Tp_allocator());
}
this->_M_impl._M_finish = this->_M_impl._M_start + __xlen;
}
return *this;
}
template<typename _Tp, typename _Alloc>
void
vector<_Tp, _Alloc>::
_M_fill_assign(size_t __n, const value_type& __val)
{
if (__n > capacity())
{
vector __tmp(__n, __val, _M_get_Tp_allocator());
__tmp._M_impl._M_swap_data(this->_M_impl);
}
else if (__n > size())
{
std::fill(begin(), end(), __val);
const size_type __add = __n - size();
;
this->_M_impl._M_finish =
std::__uninitialized_fill_n_a(this->_M_impl._M_finish,
__add, __val, _M_get_Tp_allocator());
;
}
else
_M_erase_at_end(std::fill_n(this->_M_impl._M_start, __n, __val));
}
template<typename _Tp, typename _Alloc>
template<typename _InputIterator>
void
vector<_Tp, _Alloc>::
_M_assign_aux(_InputIterator __first, _InputIterator __last,
std::input_iterator_tag)
{
pointer __cur(this->_M_impl._M_start);
for (; __first != __last && __cur != this->_M_impl._M_finish;
++__cur, (void)++__first)
*__cur = *__first;
if (__first == __last)
_M_erase_at_end(__cur);
else
_M_range_insert(end(), __first, __last,
std::__iterator_category(__first));
}
template<typename _Tp, typename _Alloc>
template<typename _ForwardIterator>
void
vector<_Tp, _Alloc>::
_M_assign_aux(_ForwardIterator __first, _ForwardIterator __last,
std::forward_iterator_tag)
{
const size_type __len = std::distance(__first, __last);
if (__len > capacity())
{
_S_check_init_len(__len, _M_get_Tp_allocator());
pointer __tmp(_M_allocate_and_copy(__len, __first, __last));
std::_Destroy(this->_M_impl._M_start, this->_M_impl._M_finish,
_M_get_Tp_allocator());
;
_M_deallocate(this->_M_impl._M_start,
this->_M_impl._M_end_of_storage
- this->_M_impl._M_start);
this->_M_impl._M_start = __tmp;
this->_M_impl._M_finish = this->_M_impl._M_start + __len;
this->_M_impl._M_end_of_storage = this->_M_impl._M_finish;
}
else if (size() >= __len)
_M_erase_at_end(std::copy(__first, __last, this->_M_impl._M_start));
else
{
_ForwardIterator __mid = __first;
std::advance(__mid, size());
std::copy(__first, __mid, this->_M_impl._M_start);
const size_type __attribute__((__unused__)) __n = __len - size();
;
this->_M_impl._M_finish =
std::__uninitialized_copy_a(__mid, __last,
this->_M_impl._M_finish,
_M_get_Tp_allocator());
;
}
}
template<typename _Tp, typename _Alloc>
auto
vector<_Tp, _Alloc>::
_M_insert_rval(const_iterator __position, value_type&& __v) -> iterator
{
const auto __n = __position - cbegin();
if (this->_M_impl._M_finish != this->_M_impl._M_end_of_storage)
if (__position == cend())
{
;
_Alloc_traits::construct(this->_M_impl, this->_M_impl._M_finish,
std::move(__v));
++this->_M_impl._M_finish;
;
}
else
_M_insert_aux(begin() + __n, std::move(__v));
else
_M_realloc_insert(begin() + __n, std::move(__v));
return iterator(this->_M_impl._M_start + __n);
}
template<typename _Tp, typename _Alloc>
template<typename... _Args>
auto
vector<_Tp, _Alloc>::
_M_emplace_aux(const_iterator __position, _Args&&... __args)
-> iterator
{
const auto __n = __position - cbegin();
if (this->_M_impl._M_finish != this->_M_impl._M_end_of_storage)
if (__position == cend())
{
;
_Alloc_traits::construct(this->_M_impl, this->_M_impl._M_finish,
std::forward<_Args>(__args)...);
++this->_M_impl._M_finish;
;
}
else
{
_Temporary_value __tmp(this, std::forward<_Args>(__args)...);
_M_insert_aux(begin() + __n, std::move(__tmp._M_val()));
}
else
_M_realloc_insert(begin() + __n, std::forward<_Args>(__args)...);
return iterator(this->_M_impl._M_start + __n);
}
template<typename _Tp, typename _Alloc>
template<typename _Arg>
void
vector<_Tp, _Alloc>::
_M_insert_aux(iterator __position, _Arg&& __arg)
{
;
_Alloc_traits::construct(this->_M_impl, this->_M_impl._M_finish,
std::move(*(this->_M_impl._M_finish - 1)));
++this->_M_impl._M_finish;
;
std::move_backward(__position.base(), this->_M_impl._M_finish - 2, this->_M_impl._M_finish - 1)
;
*__position = std::forward<_Arg>(__arg);
}
template<typename _Tp, typename _Alloc>
template<typename... _Args>
void
vector<_Tp, _Alloc>::
_M_realloc_insert(iterator __position, _Args&&... __args)
{
const size_type __len =
_M_check_len(size_type(1), "vector::_M_realloc_insert");
pointer __old_start = this->_M_impl._M_start;
pointer __old_finish = this->_M_impl._M_finish;
const size_type __elems_before = __position - begin();
pointer __new_start(this->_M_allocate(__len));
pointer __new_finish(__new_start);
try
{
_Alloc_traits::construct(this->_M_impl,
__new_start + __elems_before,
std::forward<_Args>(__args)...);
__new_finish = pointer();
if constexpr (_S_use_relocate())
{
__new_finish = _S_relocate(__old_start, __position.base(),
__new_start, _M_get_Tp_allocator());
++__new_finish;
__new_finish = _S_relocate(__position.base(), __old_finish,
__new_finish, _M_get_Tp_allocator());
}
else
{
__new_finish
= std::__uninitialized_move_if_noexcept_a
(__old_start, __position.base(),
__new_start, _M_get_Tp_allocator());
++__new_finish;
__new_finish
= std::__uninitialized_move_if_noexcept_a
(__position.base(), __old_finish,
__new_finish, _M_get_Tp_allocator());
}
}
catch(...)
{
if (!__new_finish)
_Alloc_traits::destroy(this->_M_impl,
__new_start + __elems_before);
else
std::_Destroy(__new_start, __new_finish, _M_get_Tp_allocator());
_M_deallocate(__new_start, __len);
throw;
}
if constexpr (!_S_use_relocate())
std::_Destroy(__old_start, __old_finish, _M_get_Tp_allocator());
;
_M_deallocate(__old_start,
this->_M_impl._M_end_of_storage - __old_start);
this->_M_impl._M_start = __new_start;
this->_M_impl._M_finish = __new_finish;
this->_M_impl._M_end_of_storage = __new_start + __len;
}
template<typename _Tp, typename _Alloc>
void
vector<_Tp, _Alloc>::
_M_fill_insert(iterator __position, size_type __n, const value_type& __x)
{
if (__n != 0)
{
if (size_type(this->_M_impl._M_end_of_storage
- this->_M_impl._M_finish) >= __n)
{
_Temporary_value __tmp(this, __x);
value_type& __x_copy = __tmp._M_val();
const size_type __elems_after = end() - __position;
pointer __old_finish(this->_M_impl._M_finish);
if (__elems_after > __n)
{
;
std::__uninitialized_move_a(__old_finish - __n,
__old_finish,
__old_finish,
_M_get_Tp_allocator());
this->_M_impl._M_finish += __n;
;
std::move_backward(__position.base(), __old_finish - __n, __old_finish)
;
std::fill(__position.base(), __position.base() + __n,
__x_copy);
}
else
{
;
this->_M_impl._M_finish =
std::__uninitialized_fill_n_a(__old_finish,
__n - __elems_after,
__x_copy,
_M_get_Tp_allocator());
;
std::__uninitialized_move_a(__position.base(), __old_finish,
this->_M_impl._M_finish,
_M_get_Tp_allocator());
this->_M_impl._M_finish += __elems_after;
;
std::fill(__position.base(), __old_finish, __x_copy);
}
}
else
{
pointer __old_start = this->_M_impl._M_start;
pointer __old_finish = this->_M_impl._M_finish;
const pointer __pos = __position.base();
const size_type __len =
_M_check_len(__n, "vector::_M_fill_insert");
const size_type __elems_before = __pos - __old_start;
pointer __new_start(this->_M_allocate(__len));
pointer __new_finish(__new_start);
try
{
std::__uninitialized_fill_n_a(__new_start + __elems_before,
__n, __x,
_M_get_Tp_allocator());
__new_finish = pointer();
__new_finish
= std::__uninitialized_move_if_noexcept_a
(__old_start, __pos, __new_start, _M_get_Tp_allocator());
__new_finish += __n;
__new_finish
= std::__uninitialized_move_if_noexcept_a
(__pos, __old_finish, __new_finish, _M_get_Tp_allocator());
}
catch(...)
{
if (!__new_finish)
std::_Destroy(__new_start + __elems_before,
__new_start + __elems_before + __n,
_M_get_Tp_allocator());
else
std::_Destroy(__new_start, __new_finish,
_M_get_Tp_allocator());
_M_deallocate(__new_start, __len);
throw;
}
std::_Destroy(__old_start, __old_finish, _M_get_Tp_allocator());
;
_M_deallocate(__old_start,
this->_M_impl._M_end_of_storage - __old_start);
this->_M_impl._M_start = __new_start;
this->_M_impl._M_finish = __new_finish;
this->_M_impl._M_end_of_storage = __new_start + __len;
}
}
}
template<typename _Tp, typename _Alloc>
void
vector<_Tp, _Alloc>::
_M_default_append(size_type __n)
{
if (__n != 0)
{
const size_type __size = size();
size_type __navail = size_type(this->_M_impl._M_end_of_storage
- this->_M_impl._M_finish);
if (__size > max_size() || __navail > max_size() - __size)
__builtin_unreachable();
if (__navail >= __n)
{
;
this->_M_impl._M_finish =
std::__uninitialized_default_n_a(this->_M_impl._M_finish,
__n, _M_get_Tp_allocator());
;
}
else
{
pointer __old_start = this->_M_impl._M_start;
pointer __old_finish = this->_M_impl._M_finish;
const size_type __len =
_M_check_len(__n, "vector::_M_default_append");
pointer __new_start(this->_M_allocate(__len));
if constexpr (_S_use_relocate())
{
try
{
std::__uninitialized_default_n_a(__new_start + __size,
__n, _M_get_Tp_allocator());
}
catch(...)
{
_M_deallocate(__new_start, __len);
throw;
}
_S_relocate(__old_start, __old_finish,
__new_start, _M_get_Tp_allocator());
}
else
{
pointer __destroy_from = pointer();
try
{
std::__uninitialized_default_n_a(__new_start + __size,
__n, _M_get_Tp_allocator());
__destroy_from = __new_start + __size;
std::__uninitialized_move_if_noexcept_a(
__old_start, __old_finish,
__new_start, _M_get_Tp_allocator());
}
catch(...)
{
if (__destroy_from)
std::_Destroy(__destroy_from, __destroy_from + __n,
_M_get_Tp_allocator());
_M_deallocate(__new_start, __len);
throw;
}
std::_Destroy(__old_start, __old_finish,
_M_get_Tp_allocator());
}
;
_M_deallocate(__old_start,
this->_M_impl._M_end_of_storage - __old_start);
this->_M_impl._M_start = __new_start;
this->_M_impl._M_finish = __new_start + __size + __n;
this->_M_impl._M_end_of_storage = __new_start + __len;
}
}
}
template<typename _Tp, typename _Alloc>
bool
vector<_Tp, _Alloc>::
_M_shrink_to_fit()
{
if (capacity() == size())
return false;
;
return std::__shrink_to_fit_aux<vector>::_S_do_it(*this);
}
template<typename _Tp, typename _Alloc>
template<typename _InputIterator>
void
vector<_Tp, _Alloc>::
_M_range_insert(iterator __pos, _InputIterator __first,
_InputIterator __last, std::input_iterator_tag)
{
if (__pos == end())
{
for (; __first != __last; ++__first)
insert(end(), *__first);
}
else if (__first != __last)
{
vector __tmp(__first, __last, _M_get_Tp_allocator());
insert(__pos,
std::make_move_iterator(__tmp.begin()),
std::make_move_iterator(__tmp.end()));
}
}
template<typename _Tp, typename _Alloc>
template<typename _ForwardIterator>
void
vector<_Tp, _Alloc>::
_M_range_insert(iterator __position, _ForwardIterator __first,
_ForwardIterator __last, std::forward_iterator_tag)
{
if (__first != __last)
{
const size_type __n = std::distance(__first, __last);
if (size_type(this->_M_impl._M_end_of_storage
- this->_M_impl._M_finish) >= __n)
{
const size_type __elems_after = end() - __position;
pointer __old_finish(this->_M_impl._M_finish);
if (__elems_after > __n)
{
;
std::__uninitialized_move_a(this->_M_impl._M_finish - __n,
this->_M_impl._M_finish,
this->_M_impl._M_finish,
_M_get_Tp_allocator());
this->_M_impl._M_finish += __n;
;
std::move_backward(__position.base(), __old_finish - __n, __old_finish)
;
std::copy(__first, __last, __position);
}
else
{
_ForwardIterator __mid = __first;
std::advance(__mid, __elems_after);
;
std::__uninitialized_copy_a(__mid, __last,
this->_M_impl._M_finish,
_M_get_Tp_allocator());
this->_M_impl._M_finish += __n - __elems_after;
;
std::__uninitialized_move_a(__position.base(),
__old_finish,
this->_M_impl._M_finish,
_M_get_Tp_allocator());
this->_M_impl._M_finish += __elems_after;
;
std::copy(__first, __mid, __position);
}
}
else
{
pointer __old_start = this->_M_impl._M_start;
pointer __old_finish = this->_M_impl._M_finish;
const size_type __len =
_M_check_len(__n, "vector::_M_range_insert");
pointer __new_start(this->_M_allocate(__len));
pointer __new_finish(__new_start);
try
{
__new_finish
= std::__uninitialized_move_if_noexcept_a
(__old_start, __position.base(),
__new_start, _M_get_Tp_allocator());
__new_finish
= std::__uninitialized_copy_a(__first, __last,
__new_finish,
_M_get_Tp_allocator());
__new_finish
= std::__uninitialized_move_if_noexcept_a
(__position.base(), __old_finish,
__new_finish, _M_get_Tp_allocator());
}
catch(...)
{
std::_Destroy(__new_start, __new_finish,
_M_get_Tp_allocator());
_M_deallocate(__new_start, __len);
throw;
}
std::_Destroy(__old_start, __old_finish,
_M_get_Tp_allocator());
;
_M_deallocate(__old_start,
this->_M_impl._M_end_of_storage - __old_start);
this->_M_impl._M_start = __new_start;
this->_M_impl._M_finish = __new_finish;
this->_M_impl._M_end_of_storage = __new_start + __len;
}
}
}
template<typename _Alloc>
void
vector<bool, _Alloc>::
_M_reallocate(size_type __n)
{
_Bit_pointer __q = this->_M_allocate(__n);
iterator __start(std::__addressof(*__q), 0);
iterator __finish(_M_copy_aligned(begin(), end(), __start));
this->_M_deallocate();
this->_M_impl._M_start = __start;
this->_M_impl._M_finish = __finish;
this->_M_impl._M_end_of_storage = __q + _S_nword(__n);
}
template<typename _Alloc>
void
vector<bool, _Alloc>::
_M_fill_insert(iterator __position, size_type __n, bool __x)
{
if (__n == 0)
return;
if (capacity() - size() >= __n)
{
std::copy_backward(__position, end(),
this->_M_impl._M_finish + difference_type(__n));
std::fill(__position, __position + difference_type(__n), __x);
this->_M_impl._M_finish += difference_type(__n);
}
else
{
const size_type __len =
_M_check_len(__n, "vector<bool>::_M_fill_insert");
_Bit_pointer __q = this->_M_allocate(__len);
iterator __start(std::__addressof(*__q), 0);
iterator __i = _M_copy_aligned(begin(), __position, __start);
std::fill(__i, __i + difference_type(__n), __x);
iterator __finish = std::copy(__position, end(),
__i + difference_type(__n));
this->_M_deallocate();
this->_M_impl._M_end_of_storage = __q + _S_nword(__len);
this->_M_impl._M_start = __start;
this->_M_impl._M_finish = __finish;
}
}
template<typename _Alloc>
template<typename _ForwardIterator>
void
vector<bool, _Alloc>::
_M_insert_range(iterator __position, _ForwardIterator __first,
_ForwardIterator __last, std::forward_iterator_tag)
{
if (__first != __last)
{
size_type __n = std::distance(__first, __last);
if (capacity() - size() >= __n)
{
std::copy_backward(__position, end(),
this->_M_impl._M_finish
+ difference_type(__n));
std::copy(__first, __last, __position);
this->_M_impl._M_finish += difference_type(__n);
}
else
{
const size_type __len =
_M_check_len(__n, "vector<bool>::_M_insert_range");
const iterator __begin = begin(), __end = end();
_Bit_pointer __q = this->_M_allocate(__len);
iterator __start(std::__addressof(*__q), 0);
iterator __i = _M_copy_aligned(__begin, __position, __start);
__i = std::copy(__first, __last, __i);
iterator __finish = std::copy(__position, __end, __i);
this->_M_deallocate();
this->_M_impl._M_end_of_storage = __q + _S_nword(__len);
this->_M_impl._M_start = __start;
this->_M_impl._M_finish = __finish;
}
}
}
template<typename _Alloc>
void
vector<bool, _Alloc>::
_M_insert_aux(iterator __position, bool __x)
{
if (this->_M_impl._M_finish._M_p != this->_M_impl._M_end_addr())
{
std::copy_backward(__position, this->_M_impl._M_finish,
this->_M_impl._M_finish + 1);
*__position = __x;
++this->_M_impl._M_finish;
}
else
{
const size_type __len =
_M_check_len(size_type(1), "vector<bool>::_M_insert_aux");
_Bit_pointer __q = this->_M_allocate(__len);
iterator __start(std::__addressof(*__q), 0);
iterator __i = _M_copy_aligned(begin(), __position, __start);
*__i++ = __x;
iterator __finish = std::copy(__position, end(), __i);
this->_M_deallocate();
this->_M_impl._M_end_of_storage = __q + _S_nword(__len);
this->_M_impl._M_start = __start;
this->_M_impl._M_finish = __finish;
}
}
template<typename _Alloc>
typename vector<bool, _Alloc>::iterator
vector<bool, _Alloc>::
_M_erase(iterator __position)
{
if (__position + 1 != end())
std::copy(__position + 1, end(), __position);
--this->_M_impl._M_finish;
return __position;
}
template<typename _Alloc>
typename vector<bool, _Alloc>::iterator
vector<bool, _Alloc>::
_M_erase(iterator __first, iterator __last)
{
if (__first != __last)
_M_erase_at_end(std::copy(__last, end(), __first));
return __first;
}
template<typename _Alloc>
bool
vector<bool, _Alloc>::
_M_shrink_to_fit()
{
if (capacity() - size() < int(_S_word_bit))
return false;
try
{
if (size_type __n = size())
_M_reallocate(__n);
else
{
this->_M_deallocate();
this->_M_impl._M_reset();
}
return true;
}
catch(...)
{ return false; }
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Alloc>
size_t
hash<std::vector<bool, _Alloc>>::
operator()(const std::vector<bool, _Alloc>& __b) const noexcept
{
size_t __hash = 0;
const size_t __words = __b.size() / _S_word_bit;
if (__words)
{
const size_t __clength = __words * sizeof(_Bit_type);
__hash = std::_Hash_impl::hash(__b._M_impl._M_start._M_p, __clength);
}
const size_t __extrabits = __b.size() % _S_word_bit;
if (__extrabits)
{
_Bit_type __hiword = *__b._M_impl._M_finish._M_p;
__hiword &= ~((~static_cast<_Bit_type>(0)) << __extrabits);
const size_t __clength
= (__extrabits + 8 - 1) / 8;
if (__words)
__hash = std::_Hash_impl::hash(&__hiword, __clength, __hash);
else
__hash = std::_Hash_impl::hash(&__hiword, __clength);
}
return __hash;
}
}
extern "C++"
{
namespace std
{
using ::max_align_t;
}
namespace std
{
enum class byte : unsigned char {};
template<typename _IntegerType> struct __byte_operand { };
template<> struct __byte_operand<bool> { using __type = byte; };
template<> struct __byte_operand<char> { using __type = byte; };
template<> struct __byte_operand<signed char> { using __type = byte; };
template<> struct __byte_operand<unsigned char> { using __type = byte; };
template<> struct __byte_operand<wchar_t> { using __type = byte; };
template<> struct __byte_operand<char16_t> { using __type = byte; };
template<> struct __byte_operand<char32_t> { using __type = byte; };
template<> struct __byte_operand<short> { using __type = byte; };
template<> struct __byte_operand<unsigned short> { using __type = byte; };
template<> struct __byte_operand<int> { using __type = byte; };
template<> struct __byte_operand<unsigned int> { using __type = byte; };
template<> struct __byte_operand<long> { using __type = byte; };
template<> struct __byte_operand<unsigned long> { using __type = byte; };
template<> struct __byte_operand<long long> { using __type = byte; };
template<> struct __byte_operand<unsigned long long> { using __type = byte; };
template<typename _IntegerType>
struct __byte_operand<const _IntegerType>
: __byte_operand<_IntegerType> { };
template<typename _IntegerType>
struct __byte_operand<volatile _IntegerType>
: __byte_operand<_IntegerType> { };
template<typename _IntegerType>
struct __byte_operand<const volatile _IntegerType>
: __byte_operand<_IntegerType> { };
template<typename _IntegerType>
using __byte_op_t = typename __byte_operand<_IntegerType>::__type;
template<typename _IntegerType>
[[__gnu__::__always_inline__]]
constexpr __byte_op_t<_IntegerType>
operator<<(byte __b, _IntegerType __shift) noexcept
{ return (byte)(unsigned char)((unsigned)__b << __shift); }
template<typename _IntegerType>
[[__gnu__::__always_inline__]]
constexpr __byte_op_t<_IntegerType>
operator>>(byte __b, _IntegerType __shift) noexcept
{ return (byte)(unsigned char)((unsigned)__b >> __shift); }
[[__gnu__::__always_inline__]]
constexpr byte
operator|(byte __l, byte __r) noexcept
{ return (byte)(unsigned char)((unsigned)__l | (unsigned)__r); }
[[__gnu__::__always_inline__]]
constexpr byte
operator&(byte __l, byte __r) noexcept
{ return (byte)(unsigned char)((unsigned)__l & (unsigned)__r); }
[[__gnu__::__always_inline__]]
constexpr byte
operator^(byte __l, byte __r) noexcept
{ return (byte)(unsigned char)((unsigned)__l ^ (unsigned)__r); }
[[__gnu__::__always_inline__]]
constexpr byte
operator~(byte __b) noexcept
{ return (byte)(unsigned char)~(unsigned)__b; }
template<typename _IntegerType>
[[__gnu__::__always_inline__]]
constexpr __byte_op_t<_IntegerType>&
operator<<=(byte& __b, _IntegerType __shift) noexcept
{ return __b = __b << __shift; }
template<typename _IntegerType>
[[__gnu__::__always_inline__]]
constexpr __byte_op_t<_IntegerType>&
operator>>=(byte& __b, _IntegerType __shift) noexcept
{ return __b = __b >> __shift; }
[[__gnu__::__always_inline__]]
constexpr byte&
operator|=(byte& __l, byte __r) noexcept
{ return __l = __l | __r; }
[[__gnu__::__always_inline__]]
constexpr byte&
operator&=(byte& __l, byte __r) noexcept
{ return __l = __l & __r; }
[[__gnu__::__always_inline__]]
constexpr byte&
operator^=(byte& __l, byte __r) noexcept
{ return __l = __l ^ __r; }
template<typename _IntegerType>
[[nodiscard,__gnu__::__always_inline__]]
constexpr _IntegerType
to_integer(__byte_op_t<_IntegerType> __b) noexcept
{ return _IntegerType(__b); }
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
struct __erased_type { };
template<typename _Alloc, typename _Tp>
using __is_erased_or_convertible
= __or_<is_convertible<_Alloc, _Tp>, is_same<_Tp, __erased_type>>;
struct allocator_arg_t { explicit allocator_arg_t() = default; };
inline constexpr allocator_arg_t allocator_arg =
allocator_arg_t();
template<typename _Tp, typename _Alloc, typename = __void_t<>>
struct __uses_allocator_helper
: false_type { };
template<typename _Tp, typename _Alloc>
struct __uses_allocator_helper<_Tp, _Alloc,
__void_t<typename _Tp::allocator_type>>
: __is_erased_or_convertible<_Alloc, typename _Tp::allocator_type>::type
{ };
template<typename _Tp, typename _Alloc>
struct uses_allocator
: __uses_allocator_helper<_Tp, _Alloc>::type
{ };
struct __uses_alloc_base { };
struct __uses_alloc0 : __uses_alloc_base
{
struct _Sink { void operator=(const void*) { } } _M_a;
};
template<typename _Alloc>
struct __uses_alloc1 : __uses_alloc_base { const _Alloc* _M_a; };
template<typename _Alloc>
struct __uses_alloc2 : __uses_alloc_base { const _Alloc* _M_a; };
template<bool, typename _Tp, typename _Alloc, typename... _Args>
struct __uses_alloc;
template<typename _Tp, typename _Alloc, typename... _Args>
struct __uses_alloc<true, _Tp, _Alloc, _Args...>
: __conditional_t<
is_constructible<_Tp, allocator_arg_t, const _Alloc&, _Args...>::value,
__uses_alloc1<_Alloc>,
__uses_alloc2<_Alloc>>
{
static_assert(__or_<
is_constructible<_Tp, allocator_arg_t, const _Alloc&, _Args...>,
is_constructible<_Tp, _Args..., const _Alloc&>>::value,
"construction with an allocator must be possible"
" if uses_allocator is true");
};
template<typename _Tp, typename _Alloc, typename... _Args>
struct __uses_alloc<false, _Tp, _Alloc, _Args...>
: __uses_alloc0 { };
template<typename _Tp, typename _Alloc, typename... _Args>
using __uses_alloc_t =
__uses_alloc<uses_allocator<_Tp, _Alloc>::value, _Tp, _Alloc, _Args...>;
template<typename _Tp, typename _Alloc, typename... _Args>
inline __uses_alloc_t<_Tp, _Alloc, _Args...>
__use_alloc(const _Alloc& __a)
{
__uses_alloc_t<_Tp, _Alloc, _Args...> __ret;
__ret._M_a = std::__addressof(__a);
return __ret;
}
template<typename _Tp, typename _Alloc, typename... _Args>
void
__use_alloc(const _Alloc&&) = delete;
template <typename _Tp, typename _Alloc>
inline constexpr bool uses_allocator_v =
uses_allocator<_Tp, _Alloc>::value;
template<template<typename...> class _Predicate,
typename _Tp, typename _Alloc, typename... _Args>
struct __is_uses_allocator_predicate
: __conditional_t<uses_allocator<_Tp, _Alloc>::value,
__or_<_Predicate<_Tp, allocator_arg_t, _Alloc, _Args...>,
_Predicate<_Tp, _Args..., _Alloc>>,
_Predicate<_Tp, _Args...>> { };
template<typename _Tp, typename _Alloc, typename... _Args>
struct __is_uses_allocator_constructible
: __is_uses_allocator_predicate<is_constructible, _Tp, _Alloc, _Args...>
{ };
template<typename _Tp, typename _Alloc, typename... _Args>
inline constexpr bool __is_uses_allocator_constructible_v =
__is_uses_allocator_constructible<_Tp, _Alloc, _Args...>::value;
template<typename _Tp, typename _Alloc, typename... _Args>
struct __is_nothrow_uses_allocator_constructible
: __is_uses_allocator_predicate<is_nothrow_constructible,
_Tp, _Alloc, _Args...>
{ };
template<typename _Tp, typename _Alloc, typename... _Args>
inline constexpr bool
__is_nothrow_uses_allocator_constructible_v =
__is_nothrow_uses_allocator_constructible<_Tp, _Alloc, _Args...>::value;
template<typename _Tp, typename... _Args>
void __uses_allocator_construct_impl(__uses_alloc0 __a, _Tp* __ptr,
_Args&&... __args)
{ ::new ((void*)__ptr) _Tp(std::forward<_Args>(__args)...); }
template<typename _Tp, typename _Alloc, typename... _Args>
void __uses_allocator_construct_impl(__uses_alloc1<_Alloc> __a, _Tp* __ptr,
_Args&&... __args)
{
::new ((void*)__ptr) _Tp(allocator_arg, *__a._M_a,
std::forward<_Args>(__args)...);
}
template<typename _Tp, typename _Alloc, typename... _Args>
void __uses_allocator_construct_impl(__uses_alloc2<_Alloc> __a, _Tp* __ptr,
_Args&&... __args)
{ ::new ((void*)__ptr) _Tp(std::forward<_Args>(__args)..., *__a._M_a); }
template<typename _Tp, typename _Alloc, typename... _Args>
void __uses_allocator_construct(const _Alloc& __a, _Tp* __ptr,
_Args&&... __args)
{
std::__uses_allocator_construct_impl(
std::__use_alloc<_Tp, _Alloc, _Args...>(__a), __ptr,
std::forward<_Args>(__args)...);
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename... _Elements>
class tuple;
template<typename _Tp>
struct __is_empty_non_tuple : is_empty<_Tp> { };
template<typename _El0, typename... _El>
struct __is_empty_non_tuple<tuple<_El0, _El...>> : false_type { };
template<typename _Tp>
using __empty_not_final
= __conditional_t<__is_final(_Tp), false_type,
__is_empty_non_tuple<_Tp>>;
template<size_t _Idx, typename _Head,
bool = __empty_not_final<_Head>::value>
struct _Head_base;
template<size_t _Idx, typename _Head>
struct _Head_base<_Idx, _Head, true>
{
constexpr _Head_base()
: _M_head_impl() { }
constexpr _Head_base(const _Head& __h)
: _M_head_impl(__h) { }
constexpr _Head_base(const _Head_base&) = default;
constexpr _Head_base(_Head_base&&) = default;
template<typename _UHead>
constexpr _Head_base(_UHead&& __h)
: _M_head_impl(std::forward<_UHead>(__h)) { }
_Head_base(allocator_arg_t, __uses_alloc0)
: _M_head_impl() { }
template<typename _Alloc>
_Head_base(allocator_arg_t, __uses_alloc1<_Alloc> __a)
: _M_head_impl(allocator_arg, *__a._M_a) { }
template<typename _Alloc>
_Head_base(allocator_arg_t, __uses_alloc2<_Alloc> __a)
: _M_head_impl(*__a._M_a) { }
template<typename _UHead>
_Head_base(__uses_alloc0, _UHead&& __uhead)
: _M_head_impl(std::forward<_UHead>(__uhead)) { }
template<typename _Alloc, typename _UHead>
_Head_base(__uses_alloc1<_Alloc> __a, _UHead&& __uhead)
: _M_head_impl(allocator_arg, *__a._M_a, std::forward<_UHead>(__uhead))
{ }
template<typename _Alloc, typename _UHead>
_Head_base(__uses_alloc2<_Alloc> __a, _UHead&& __uhead)
: _M_head_impl(std::forward<_UHead>(__uhead), *__a._M_a) { }
static constexpr _Head&
_M_head(_Head_base& __b) noexcept { return __b._M_head_impl; }
static constexpr const _Head&
_M_head(const _Head_base& __b) noexcept { return __b._M_head_impl; }
[[__no_unique_address__]] _Head _M_head_impl;
};
template<size_t _Idx, typename _Head>
struct _Head_base<_Idx, _Head, false>
{
constexpr _Head_base()
: _M_head_impl() { }
constexpr _Head_base(const _Head& __h)
: _M_head_impl(__h) { }
constexpr _Head_base(const _Head_base&) = default;
constexpr _Head_base(_Head_base&&) = default;
template<typename _UHead>
constexpr _Head_base(_UHead&& __h)
: _M_head_impl(std::forward<_UHead>(__h)) { }
_Head_base(allocator_arg_t, __uses_alloc0)
: _M_head_impl() { }
template<typename _Alloc>
_Head_base(allocator_arg_t, __uses_alloc1<_Alloc> __a)
: _M_head_impl(allocator_arg, *__a._M_a) { }
template<typename _Alloc>
_Head_base(allocator_arg_t, __uses_alloc2<_Alloc> __a)
: _M_head_impl(*__a._M_a) { }
template<typename _UHead>
_Head_base(__uses_alloc0, _UHead&& __uhead)
: _M_head_impl(std::forward<_UHead>(__uhead)) { }
template<typename _Alloc, typename _UHead>
_Head_base(__uses_alloc1<_Alloc> __a, _UHead&& __uhead)
: _M_head_impl(allocator_arg, *__a._M_a, std::forward<_UHead>(__uhead))
{ }
template<typename _Alloc, typename _UHead>
_Head_base(__uses_alloc2<_Alloc> __a, _UHead&& __uhead)
: _M_head_impl(std::forward<_UHead>(__uhead), *__a._M_a) { }
static constexpr _Head&
_M_head(_Head_base& __b) noexcept { return __b._M_head_impl; }
static constexpr const _Head&
_M_head(const _Head_base& __b) noexcept { return __b._M_head_impl; }
_Head _M_head_impl;
};
template<size_t _Idx, typename... _Elements>
struct _Tuple_impl;
template<size_t _Idx, typename _Head, typename... _Tail>
struct _Tuple_impl<_Idx, _Head, _Tail...>
: public _Tuple_impl<_Idx + 1, _Tail...>,
private _Head_base<_Idx, _Head>
{
template<size_t, typename...> friend struct _Tuple_impl;
typedef _Tuple_impl<_Idx + 1, _Tail...> _Inherited;
typedef _Head_base<_Idx, _Head> _Base;
static constexpr _Head&
_M_head(_Tuple_impl& __t) noexcept { return _Base::_M_head(__t); }
static constexpr const _Head&
_M_head(const _Tuple_impl& __t) noexcept { return _Base::_M_head(__t); }
static constexpr _Inherited&
_M_tail(_Tuple_impl& __t) noexcept { return __t; }
static constexpr const _Inherited&
_M_tail(const _Tuple_impl& __t) noexcept { return __t; }
constexpr _Tuple_impl()
: _Inherited(), _Base() { }
explicit constexpr
_Tuple_impl(const _Head& __head, const _Tail&... __tail)
: _Inherited(__tail...), _Base(__head)
{ }
template<typename _UHead, typename... _UTail,
typename = __enable_if_t<sizeof...(_Tail) == sizeof...(_UTail)>>
explicit constexpr
_Tuple_impl(_UHead&& __head, _UTail&&... __tail)
: _Inherited(std::forward<_UTail>(__tail)...),
_Base(std::forward<_UHead>(__head))
{ }
constexpr _Tuple_impl(const _Tuple_impl&) = default;
_Tuple_impl& operator=(const _Tuple_impl&) = delete;
_Tuple_impl(_Tuple_impl&&) = default;
template<typename... _UElements>
constexpr
_Tuple_impl(const _Tuple_impl<_Idx, _UElements...>& __in)
: _Inherited(_Tuple_impl<_Idx, _UElements...>::_M_tail(__in)),
_Base(_Tuple_impl<_Idx, _UElements...>::_M_head(__in))
{ }
template<typename _UHead, typename... _UTails>
constexpr
_Tuple_impl(_Tuple_impl<_Idx, _UHead, _UTails...>&& __in)
: _Inherited(std::move
(_Tuple_impl<_Idx, _UHead, _UTails...>::_M_tail(__in))),
_Base(std::forward<_UHead>
(_Tuple_impl<_Idx, _UHead, _UTails...>::_M_head(__in)))
{ }
template<typename _Alloc>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a)
: _Inherited(__tag, __a),
_Base(__tag, __use_alloc<_Head>(__a))
{ }
template<typename _Alloc>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
const _Head& __head, const _Tail&... __tail)
: _Inherited(__tag, __a, __tail...),
_Base(__use_alloc<_Head, _Alloc, _Head>(__a), __head)
{ }
template<typename _Alloc, typename _UHead, typename... _UTail,
typename = __enable_if_t<sizeof...(_Tail) == sizeof...(_UTail)>>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
_UHead&& __head, _UTail&&... __tail)
: _Inherited(__tag, __a, std::forward<_UTail>(__tail)...),
_Base(__use_alloc<_Head, _Alloc, _UHead>(__a),
std::forward<_UHead>(__head))
{ }
template<typename _Alloc>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
const _Tuple_impl& __in)
: _Inherited(__tag, __a, _M_tail(__in)),
_Base(__use_alloc<_Head, _Alloc, _Head>(__a), _M_head(__in))
{ }
template<typename _Alloc>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
_Tuple_impl&& __in)
: _Inherited(__tag, __a, std::move(_M_tail(__in))),
_Base(__use_alloc<_Head, _Alloc, _Head>(__a),
std::forward<_Head>(_M_head(__in)))
{ }
template<typename _Alloc, typename _UHead, typename... _UTails>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
const _Tuple_impl<_Idx, _UHead, _UTails...>& __in)
: _Inherited(__tag, __a,
_Tuple_impl<_Idx, _UHead, _UTails...>::_M_tail(__in)),
_Base(__use_alloc<_Head, _Alloc, const _UHead&>(__a),
_Tuple_impl<_Idx, _UHead, _UTails...>::_M_head(__in))
{ }
template<typename _Alloc, typename _UHead, typename... _UTails>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
_Tuple_impl<_Idx, _UHead, _UTails...>&& __in)
: _Inherited(__tag, __a, std::move
(_Tuple_impl<_Idx, _UHead, _UTails...>::_M_tail(__in))),
_Base(__use_alloc<_Head, _Alloc, _UHead>(__a),
std::forward<_UHead>
(_Tuple_impl<_Idx, _UHead, _UTails...>::_M_head(__in)))
{ }
template<typename... _UElements>
void
_M_assign(const _Tuple_impl<_Idx, _UElements...>& __in)
{
_M_head(*this) = _Tuple_impl<_Idx, _UElements...>::_M_head(__in);
_M_tail(*this)._M_assign(
_Tuple_impl<_Idx, _UElements...>::_M_tail(__in));
}
template<typename _UHead, typename... _UTails>
void
_M_assign(_Tuple_impl<_Idx, _UHead, _UTails...>&& __in)
{
_M_head(*this) = std::forward<_UHead>
(_Tuple_impl<_Idx, _UHead, _UTails...>::_M_head(__in));
_M_tail(*this)._M_assign(
std::move(_Tuple_impl<_Idx, _UHead, _UTails...>::_M_tail(__in)));
}
protected:
void
_M_swap(_Tuple_impl& __in)
{
using std::swap;
swap(_M_head(*this), _M_head(__in));
_Inherited::_M_swap(_M_tail(__in));
}
};
template<size_t _Idx, typename _Head>
struct _Tuple_impl<_Idx, _Head>
: private _Head_base<_Idx, _Head>
{
template<size_t, typename...> friend struct _Tuple_impl;
typedef _Head_base<_Idx, _Head> _Base;
static constexpr _Head&
_M_head(_Tuple_impl& __t) noexcept { return _Base::_M_head(__t); }
static constexpr const _Head&
_M_head(const _Tuple_impl& __t) noexcept { return _Base::_M_head(__t); }
constexpr
_Tuple_impl()
: _Base() { }
explicit constexpr
_Tuple_impl(const _Head& __head)
: _Base(__head)
{ }
template<typename _UHead>
explicit constexpr
_Tuple_impl(_UHead&& __head)
: _Base(std::forward<_UHead>(__head))
{ }
constexpr _Tuple_impl(const _Tuple_impl&) = default;
_Tuple_impl& operator=(const _Tuple_impl&) = delete;
constexpr
_Tuple_impl(_Tuple_impl&& __in)
noexcept(is_nothrow_move_constructible<_Head>::value)
: _Base(static_cast<_Base&&>(__in))
{ }
template<typename _UHead>
constexpr
_Tuple_impl(const _Tuple_impl<_Idx, _UHead>& __in)
: _Base(_Tuple_impl<_Idx, _UHead>::_M_head(__in))
{ }
template<typename _UHead>
constexpr
_Tuple_impl(_Tuple_impl<_Idx, _UHead>&& __in)
: _Base(std::forward<_UHead>(_Tuple_impl<_Idx, _UHead>::_M_head(__in)))
{ }
template<typename _Alloc>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a)
: _Base(__tag, __use_alloc<_Head>(__a))
{ }
template<typename _Alloc>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
const _Head& __head)
: _Base(__use_alloc<_Head, _Alloc, const _Head&>(__a), __head)
{ }
template<typename _Alloc, typename _UHead>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
_UHead&& __head)
: _Base(__use_alloc<_Head, _Alloc, _UHead>(__a),
std::forward<_UHead>(__head))
{ }
template<typename _Alloc>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
const _Tuple_impl& __in)
: _Base(__use_alloc<_Head, _Alloc, const _Head&>(__a), _M_head(__in))
{ }
template<typename _Alloc>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
_Tuple_impl&& __in)
: _Base(__use_alloc<_Head, _Alloc, _Head>(__a),
std::forward<_Head>(_M_head(__in)))
{ }
template<typename _Alloc, typename _UHead>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
const _Tuple_impl<_Idx, _UHead>& __in)
: _Base(__use_alloc<_Head, _Alloc, const _UHead&>(__a),
_Tuple_impl<_Idx, _UHead>::_M_head(__in))
{ }
template<typename _Alloc, typename _UHead>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
_Tuple_impl<_Idx, _UHead>&& __in)
: _Base(__use_alloc<_Head, _Alloc, _UHead>(__a),
std::forward<_UHead>(_Tuple_impl<_Idx, _UHead>::_M_head(__in)))
{ }
template<typename _UHead>
void
_M_assign(const _Tuple_impl<_Idx, _UHead>& __in)
{
_M_head(*this) = _Tuple_impl<_Idx, _UHead>::_M_head(__in);
}
template<typename _UHead>
void
_M_assign(_Tuple_impl<_Idx, _UHead>&& __in)
{
_M_head(*this)
= std::forward<_UHead>(_Tuple_impl<_Idx, _UHead>::_M_head(__in));
}
protected:
void
_M_swap(_Tuple_impl& __in)
{
using std::swap;
swap(_M_head(*this), _M_head(__in));
}
};
template<bool, typename... _Types>
struct _TupleConstraints
{
template<typename... _UTypes>
using __constructible = __and_<is_constructible<_Types, _UTypes>...>;
template<typename... _UTypes>
using __convertible = __and_<is_convertible<_UTypes, _Types>...>;
template<typename... _UTypes>
static constexpr bool __is_implicitly_constructible()
{
return __and_<__constructible<_UTypes...>,
__convertible<_UTypes...>
>::value;
}
template<typename... _UTypes>
static constexpr bool __is_explicitly_constructible()
{
return __and_<__constructible<_UTypes...>,
__not_<__convertible<_UTypes...>>
>::value;
}
static constexpr bool __is_implicitly_default_constructible()
{
return __and_<std::__is_implicitly_default_constructible<_Types>...
>::value;
}
static constexpr bool __is_explicitly_default_constructible()
{
return __and_<is_default_constructible<_Types>...,
__not_<__and_<
std::__is_implicitly_default_constructible<_Types>...>
>>::value;
}
};
template<typename... _Types>
struct _TupleConstraints<false, _Types...>
{
template<typename... _UTypes>
static constexpr bool __is_implicitly_constructible()
{ return false; }
template<typename... _UTypes>
static constexpr bool __is_explicitly_constructible()
{ return false; }
};
template<typename... _Elements>
class tuple : public _Tuple_impl<0, _Elements...>
{
typedef _Tuple_impl<0, _Elements...> _Inherited;
template<bool _Cond>
using _TCC = _TupleConstraints<_Cond, _Elements...>;
template<bool _Dummy>
using _ImplicitDefaultCtor = __enable_if_t<
_TCC<_Dummy>::__is_implicitly_default_constructible(),
bool>;
template<bool _Dummy>
using _ExplicitDefaultCtor = __enable_if_t<
_TCC<_Dummy>::__is_explicitly_default_constructible(),
bool>;
template<bool _Cond, typename... _Args>
using _ImplicitCtor = __enable_if_t<
_TCC<_Cond>::template __is_implicitly_constructible<_Args...>(),
bool>;
template<bool _Cond, typename... _Args>
using _ExplicitCtor = __enable_if_t<
_TCC<_Cond>::template __is_explicitly_constructible<_Args...>(),
bool>;
template<typename... _UElements>
static constexpr
__enable_if_t<sizeof...(_UElements) == sizeof...(_Elements), bool>
__assignable()
{ return __and_<is_assignable<_Elements&, _UElements>...>::value; }
template<typename... _UElements>
static constexpr bool __nothrow_assignable()
{
return
__and_<is_nothrow_assignable<_Elements&, _UElements>...>::value;
}
template<typename... _UElements>
static constexpr bool __nothrow_constructible()
{
return
__and_<is_nothrow_constructible<_Elements, _UElements>...>::value;
}
template<typename _Up>
static constexpr bool __valid_args()
{
return sizeof...(_Elements) == 1
&& !is_same<tuple, __remove_cvref_t<_Up>>::value;
}
template<typename, typename, typename... _Tail>
static constexpr bool __valid_args()
{ return (sizeof...(_Tail) + 2) == sizeof...(_Elements); }
template<typename _Tuple, typename = tuple,
typename = __remove_cvref_t<_Tuple>>
struct _UseOtherCtor
: false_type
{ };
template<typename _Tuple, typename _Tp, typename _Up>
struct _UseOtherCtor<_Tuple, tuple<_Tp>, tuple<_Up>>
: __or_<is_convertible<_Tuple, _Tp>, is_constructible<_Tp, _Tuple>>::type
{ };
template<typename _Tuple, typename _Tp>
struct _UseOtherCtor<_Tuple, tuple<_Tp>, tuple<_Tp>>
: true_type
{ };
template<typename _Tuple>
static constexpr bool __use_other_ctor()
{ return _UseOtherCtor<_Tuple>::value; }
public:
template<typename _Dummy = void,
_ImplicitDefaultCtor<is_void<_Dummy>::value> = true>
constexpr
tuple()
noexcept(__and_<is_nothrow_default_constructible<_Elements>...>::value)
: _Inherited() { }
template<typename _Dummy = void,
_ExplicitDefaultCtor<is_void<_Dummy>::value> = false>
explicit constexpr
tuple()
noexcept(__and_<is_nothrow_default_constructible<_Elements>...>::value)
: _Inherited() { }
template<bool _NotEmpty = (sizeof...(_Elements) >= 1),
_ImplicitCtor<_NotEmpty, const _Elements&...> = true>
constexpr
tuple(const _Elements&... __elements)
noexcept(__nothrow_constructible<const _Elements&...>())
: _Inherited(__elements...) { }
template<bool _NotEmpty = (sizeof...(_Elements) >= 1),
_ExplicitCtor<_NotEmpty, const _Elements&...> = false>
explicit constexpr
tuple(const _Elements&... __elements)
noexcept(__nothrow_constructible<const _Elements&...>())
: _Inherited(__elements...) { }
template<typename... _UElements,
bool _Valid = __valid_args<_UElements...>(),
_ImplicitCtor<_Valid, _UElements...> = true>
constexpr
tuple(_UElements&&... __elements)
noexcept(__nothrow_constructible<_UElements...>())
: _Inherited(std::forward<_UElements>(__elements)...) { }
template<typename... _UElements,
bool _Valid = __valid_args<_UElements...>(),
_ExplicitCtor<_Valid, _UElements...> = false>
explicit constexpr
tuple(_UElements&&... __elements)
noexcept(__nothrow_constructible<_UElements...>())
: _Inherited(std::forward<_UElements>(__elements)...) { }
constexpr tuple(const tuple&) = default;
constexpr tuple(tuple&&) = default;
template<typename... _UElements,
bool _Valid = (sizeof...(_Elements) == sizeof...(_UElements))
&& !__use_other_ctor<const tuple<_UElements...>&>(),
_ImplicitCtor<_Valid, const _UElements&...> = true>
constexpr
tuple(const tuple<_UElements...>& __in)
noexcept(__nothrow_constructible<const _UElements&...>())
: _Inherited(static_cast<const _Tuple_impl<0, _UElements...>&>(__in))
{ }
template<typename... _UElements,
bool _Valid = (sizeof...(_Elements) == sizeof...(_UElements))
&& !__use_other_ctor<const tuple<_UElements...>&>(),
_ExplicitCtor<_Valid, const _UElements&...> = false>
explicit constexpr
tuple(const tuple<_UElements...>& __in)
noexcept(__nothrow_constructible<const _UElements&...>())
: _Inherited(static_cast<const _Tuple_impl<0, _UElements...>&>(__in))
{ }
template<typename... _UElements,
bool _Valid = (sizeof...(_Elements) == sizeof...(_UElements))
&& !__use_other_ctor<tuple<_UElements...>&&>(),
_ImplicitCtor<_Valid, _UElements...> = true>
constexpr
tuple(tuple<_UElements...>&& __in)
noexcept(__nothrow_constructible<_UElements...>())
: _Inherited(static_cast<_Tuple_impl<0, _UElements...>&&>(__in)) { }
template<typename... _UElements,
bool _Valid = (sizeof...(_Elements) == sizeof...(_UElements))
&& !__use_other_ctor<tuple<_UElements...>&&>(),
_ExplicitCtor<_Valid, _UElements...> = false>
explicit constexpr
tuple(tuple<_UElements...>&& __in)
noexcept(__nothrow_constructible<_UElements...>())
: _Inherited(static_cast<_Tuple_impl<0, _UElements...>&&>(__in)) { }
template<typename _Alloc,
_ImplicitDefaultCtor<is_object<_Alloc>::value> = true>
tuple(allocator_arg_t __tag, const _Alloc& __a)
: _Inherited(__tag, __a) { }
template<typename _Alloc, bool _NotEmpty = (sizeof...(_Elements) >= 1),
_ImplicitCtor<_NotEmpty, const _Elements&...> = true>
tuple(allocator_arg_t __tag, const _Alloc& __a,
const _Elements&... __elements)
: _Inherited(__tag, __a, __elements...) { }
template<typename _Alloc, bool _NotEmpty = (sizeof...(_Elements) >= 1),
_ExplicitCtor<_NotEmpty, const _Elements&...> = false>
explicit
tuple(allocator_arg_t __tag, const _Alloc& __a,
const _Elements&... __elements)
: _Inherited(__tag, __a, __elements...) { }
template<typename _Alloc, typename... _UElements,
bool _Valid = __valid_args<_UElements...>(),
_ImplicitCtor<_Valid, _UElements...> = true>
tuple(allocator_arg_t __tag, const _Alloc& __a,
_UElements&&... __elements)
: _Inherited(__tag, __a, std::forward<_UElements>(__elements)...)
{ }
template<typename _Alloc, typename... _UElements,
bool _Valid = __valid_args<_UElements...>(),
_ExplicitCtor<_Valid, _UElements...> = false>
explicit
tuple(allocator_arg_t __tag, const _Alloc& __a,
_UElements&&... __elements)
: _Inherited(__tag, __a, std::forward<_UElements>(__elements)...)
{ }
template<typename _Alloc>
tuple(allocator_arg_t __tag, const _Alloc& __a, const tuple& __in)
: _Inherited(__tag, __a, static_cast<const _Inherited&>(__in)) { }
template<typename _Alloc>
tuple(allocator_arg_t __tag, const _Alloc& __a, tuple&& __in)
: _Inherited(__tag, __a, static_cast<_Inherited&&>(__in)) { }
template<typename _Alloc, typename... _UElements,
bool _Valid = (sizeof...(_Elements) == sizeof...(_UElements))
&& !__use_other_ctor<const tuple<_UElements...>&>(),
_ImplicitCtor<_Valid, const _UElements&...> = true>
tuple(allocator_arg_t __tag, const _Alloc& __a,
const tuple<_UElements...>& __in)
: _Inherited(__tag, __a,
static_cast<const _Tuple_impl<0, _UElements...>&>(__in))
{ }
template<typename _Alloc, typename... _UElements,
bool _Valid = (sizeof...(_Elements) == sizeof...(_UElements))
&& !__use_other_ctor<const tuple<_UElements...>&>(),
_ExplicitCtor<_Valid, const _UElements&...> = false>
explicit
tuple(allocator_arg_t __tag, const _Alloc& __a,
const tuple<_UElements...>& __in)
: _Inherited(__tag, __a,
static_cast<const _Tuple_impl<0, _UElements...>&>(__in))
{ }
template<typename _Alloc, typename... _UElements,
bool _Valid = (sizeof...(_Elements) == sizeof...(_UElements))
&& !__use_other_ctor<tuple<_UElements...>&&>(),
_ImplicitCtor<_Valid, _UElements...> = true>
tuple(allocator_arg_t __tag, const _Alloc& __a,
tuple<_UElements...>&& __in)
: _Inherited(__tag, __a,
static_cast<_Tuple_impl<0, _UElements...>&&>(__in))
{ }
template<typename _Alloc, typename... _UElements,
bool _Valid = (sizeof...(_Elements) == sizeof...(_UElements))
&& !__use_other_ctor<tuple<_UElements...>&&>(),
_ExplicitCtor<_Valid, _UElements...> = false>
explicit
tuple(allocator_arg_t __tag, const _Alloc& __a,
tuple<_UElements...>&& __in)
: _Inherited(__tag, __a,
static_cast<_Tuple_impl<0, _UElements...>&&>(__in))
{ }
tuple&
operator=(__conditional_t<__assignable<const _Elements&...>(),
const tuple&,
const __nonesuch&> __in)
noexcept(__nothrow_assignable<const _Elements&...>())
{
this->_M_assign(__in);
return *this;
}
tuple&
operator=(__conditional_t<__assignable<_Elements...>(),
tuple&&,
__nonesuch&&> __in)
noexcept(__nothrow_assignable<_Elements...>())
{
this->_M_assign(std::move(__in));
return *this;
}
template<typename... _UElements>
__enable_if_t<__assignable<const _UElements&...>(), tuple&>
operator=(const tuple<_UElements...>& __in)
noexcept(__nothrow_assignable<const _UElements&...>())
{
this->_M_assign(__in);
return *this;
}
template<typename... _UElements>
__enable_if_t<__assignable<_UElements...>(), tuple&>
operator=(tuple<_UElements...>&& __in)
noexcept(__nothrow_assignable<_UElements...>())
{
this->_M_assign(std::move(__in));
return *this;
}
void
swap(tuple& __in)
noexcept(__and_<__is_nothrow_swappable<_Elements>...>::value)
{ _Inherited::_M_swap(__in); }
};
template<typename... _UTypes>
tuple(_UTypes...) -> tuple<_UTypes...>;
template<typename _T1, typename _T2>
tuple(pair<_T1, _T2>) -> tuple<_T1, _T2>;
template<typename _Alloc, typename... _UTypes>
tuple(allocator_arg_t, _Alloc, _UTypes...) -> tuple<_UTypes...>;
template<typename _Alloc, typename _T1, typename _T2>
tuple(allocator_arg_t, _Alloc, pair<_T1, _T2>) -> tuple<_T1, _T2>;
template<typename _Alloc, typename... _UTypes>
tuple(allocator_arg_t, _Alloc, tuple<_UTypes...>) -> tuple<_UTypes...>;
template<>
class tuple<>
{
public:
void swap(tuple&) noexcept { }
tuple() = default;
template<typename _Alloc>
tuple(allocator_arg_t, const _Alloc&) noexcept { }
template<typename _Alloc>
tuple(allocator_arg_t, const _Alloc&, const tuple&) noexcept { }
};
template<typename _T1, typename _T2>
class tuple<_T1, _T2> : public _Tuple_impl<0, _T1, _T2>
{
typedef _Tuple_impl<0, _T1, _T2> _Inherited;
template<bool _Dummy, typename _U1, typename _U2>
using _ImplicitDefaultCtor = __enable_if_t<
_TupleConstraints<_Dummy, _U1, _U2>::
__is_implicitly_default_constructible(),
bool>;
template<bool _Dummy, typename _U1, typename _U2>
using _ExplicitDefaultCtor = __enable_if_t<
_TupleConstraints<_Dummy, _U1, _U2>::
__is_explicitly_default_constructible(),
bool>;
template<bool _Dummy>
using _TCC = _TupleConstraints<_Dummy, _T1, _T2>;
template<bool _Cond, typename _U1, typename _U2>
using _ImplicitCtor = __enable_if_t<
_TCC<_Cond>::template __is_implicitly_constructible<_U1, _U2>(),
bool>;
template<bool _Cond, typename _U1, typename _U2>
using _ExplicitCtor = __enable_if_t<
_TCC<_Cond>::template __is_explicitly_constructible<_U1, _U2>(),
bool>;
template<typename _U1, typename _U2>
static constexpr bool __assignable()
{
return __and_<is_assignable<_T1&, _U1>,
is_assignable<_T2&, _U2>>::value;
}
template<typename _U1, typename _U2>
static constexpr bool __nothrow_assignable()
{
return __and_<is_nothrow_assignable<_T1&, _U1>,
is_nothrow_assignable<_T2&, _U2>>::value;
}
template<typename _U1, typename _U2>
static constexpr bool __nothrow_constructible()
{
return __and_<is_nothrow_constructible<_T1, _U1>,
is_nothrow_constructible<_T2, _U2>>::value;
}
static constexpr bool __nothrow_default_constructible()
{
return __and_<is_nothrow_default_constructible<_T1>,
is_nothrow_default_constructible<_T2>>::value;
}
template<typename _U1>
static constexpr bool __is_alloc_arg()
{ return is_same<__remove_cvref_t<_U1>, allocator_arg_t>::value; }
public:
template<bool _Dummy = true,
_ImplicitDefaultCtor<_Dummy, _T1, _T2> = true>
constexpr
tuple()
noexcept(__nothrow_default_constructible())
: _Inherited() { }
template<bool _Dummy = true,
_ExplicitDefaultCtor<_Dummy, _T1, _T2> = false>
explicit constexpr
tuple()
noexcept(__nothrow_default_constructible())
: _Inherited() { }
template<bool _Dummy = true,
_ImplicitCtor<_Dummy, const _T1&, const _T2&> = true>
constexpr
tuple(const _T1& __a1, const _T2& __a2)
noexcept(__nothrow_constructible<const _T1&, const _T2&>())
: _Inherited(__a1, __a2) { }
template<bool _Dummy = true,
_ExplicitCtor<_Dummy, const _T1&, const _T2&> = false>
explicit constexpr
tuple(const _T1& __a1, const _T2& __a2)
noexcept(__nothrow_constructible<const _T1&, const _T2&>())
: _Inherited(__a1, __a2) { }
template<typename _U1, typename _U2,
_ImplicitCtor<!__is_alloc_arg<_U1>(), _U1, _U2> = true>
constexpr
tuple(_U1&& __a1, _U2&& __a2)
noexcept(__nothrow_constructible<_U1, _U2>())
: _Inherited(std::forward<_U1>(__a1), std::forward<_U2>(__a2)) { }
template<typename _U1, typename _U2,
_ExplicitCtor<!__is_alloc_arg<_U1>(), _U1, _U2> = false>
explicit constexpr
tuple(_U1&& __a1, _U2&& __a2)
noexcept(__nothrow_constructible<_U1, _U2>())
: _Inherited(std::forward<_U1>(__a1), std::forward<_U2>(__a2)) { }
constexpr tuple(const tuple&) = default;
constexpr tuple(tuple&&) = default;
template<typename _U1, typename _U2,
_ImplicitCtor<true, const _U1&, const _U2&> = true>
constexpr
tuple(const tuple<_U1, _U2>& __in)
noexcept(__nothrow_constructible<const _U1&, const _U2&>())
: _Inherited(static_cast<const _Tuple_impl<0, _U1, _U2>&>(__in)) { }
template<typename _U1, typename _U2,
_ExplicitCtor<true, const _U1&, const _U2&> = false>
explicit constexpr
tuple(const tuple<_U1, _U2>& __in)
noexcept(__nothrow_constructible<const _U1&, const _U2&>())
: _Inherited(static_cast<const _Tuple_impl<0, _U1, _U2>&>(__in)) { }
template<typename _U1, typename _U2,
_ImplicitCtor<true, _U1, _U2> = true>
constexpr
tuple(tuple<_U1, _U2>&& __in)
noexcept(__nothrow_constructible<_U1, _U2>())
: _Inherited(static_cast<_Tuple_impl<0, _U1, _U2>&&>(__in)) { }
template<typename _U1, typename _U2,
_ExplicitCtor<true, _U1, _U2> = false>
explicit constexpr
tuple(tuple<_U1, _U2>&& __in)
noexcept(__nothrow_constructible<_U1, _U2>())
: _Inherited(static_cast<_Tuple_impl<0, _U1, _U2>&&>(__in)) { }
template<typename _U1, typename _U2,
_ImplicitCtor<true, const _U1&, const _U2&> = true>
constexpr
tuple(const pair<_U1, _U2>& __in)
noexcept(__nothrow_constructible<const _U1&, const _U2&>())
: _Inherited(__in.first, __in.second) { }
template<typename _U1, typename _U2,
_ExplicitCtor<true, const _U1&, const _U2&> = false>
explicit constexpr
tuple(const pair<_U1, _U2>& __in)
noexcept(__nothrow_constructible<const _U1&, const _U2&>())
: _Inherited(__in.first, __in.second) { }
template<typename _U1, typename _U2,
_ImplicitCtor<true, _U1, _U2> = true>
constexpr
tuple(pair<_U1, _U2>&& __in)
noexcept(__nothrow_constructible<_U1, _U2>())
: _Inherited(std::forward<_U1>(__in.first),
std::forward<_U2>(__in.second)) { }
template<typename _U1, typename _U2,
_ExplicitCtor<true, _U1, _U2> = false>
explicit constexpr
tuple(pair<_U1, _U2>&& __in)
noexcept(__nothrow_constructible<_U1, _U2>())
: _Inherited(std::forward<_U1>(__in.first),
std::forward<_U2>(__in.second)) { }
template<typename _Alloc,
_ImplicitDefaultCtor<is_object<_Alloc>::value, _T1, _T2> = true>
tuple(allocator_arg_t __tag, const _Alloc& __a)
: _Inherited(__tag, __a) { }
template<typename _Alloc, bool _Dummy = true,
_ImplicitCtor<_Dummy, const _T1&, const _T2&> = true>
tuple(allocator_arg_t __tag, const _Alloc& __a,
const _T1& __a1, const _T2& __a2)
: _Inherited(__tag, __a, __a1, __a2) { }
template<typename _Alloc, bool _Dummy = true,
_ExplicitCtor<_Dummy, const _T1&, const _T2&> = false>
explicit
tuple(allocator_arg_t __tag, const _Alloc& __a,
const _T1& __a1, const _T2& __a2)
: _Inherited(__tag, __a, __a1, __a2) { }
template<typename _Alloc, typename _U1, typename _U2,
_ImplicitCtor<true, _U1, _U2> = true>
tuple(allocator_arg_t __tag, const _Alloc& __a, _U1&& __a1, _U2&& __a2)
: _Inherited(__tag, __a, std::forward<_U1>(__a1),
std::forward<_U2>(__a2)) { }
template<typename _Alloc, typename _U1, typename _U2,
_ExplicitCtor<true, _U1, _U2> = false>
explicit
tuple(allocator_arg_t __tag, const _Alloc& __a,
_U1&& __a1, _U2&& __a2)
: _Inherited(__tag, __a, std::forward<_U1>(__a1),
std::forward<_U2>(__a2)) { }
template<typename _Alloc>
tuple(allocator_arg_t __tag, const _Alloc& __a, const tuple& __in)
: _Inherited(__tag, __a, static_cast<const _Inherited&>(__in)) { }
template<typename _Alloc>
tuple(allocator_arg_t __tag, const _Alloc& __a, tuple&& __in)
: _Inherited(__tag, __a, static_cast<_Inherited&&>(__in)) { }
template<typename _Alloc, typename _U1, typename _U2,
_ImplicitCtor<true, const _U1&, const _U2&> = true>
tuple(allocator_arg_t __tag, const _Alloc& __a,
const tuple<_U1, _U2>& __in)
: _Inherited(__tag, __a,
static_cast<const _Tuple_impl<0, _U1, _U2>&>(__in))
{ }
template<typename _Alloc, typename _U1, typename _U2,
_ExplicitCtor<true, const _U1&, const _U2&> = false>
explicit
tuple(allocator_arg_t __tag, const _Alloc& __a,
const tuple<_U1, _U2>& __in)
: _Inherited(__tag, __a,
static_cast<const _Tuple_impl<0, _U1, _U2>&>(__in))
{ }
template<typename _Alloc, typename _U1, typename _U2,
_ImplicitCtor<true, _U1, _U2> = true>
tuple(allocator_arg_t __tag, const _Alloc& __a, tuple<_U1, _U2>&& __in)
: _Inherited(__tag, __a, static_cast<_Tuple_impl<0, _U1, _U2>&&>(__in))
{ }
template<typename _Alloc, typename _U1, typename _U2,
_ExplicitCtor<true, _U1, _U2> = false>
explicit
tuple(allocator_arg_t __tag, const _Alloc& __a, tuple<_U1, _U2>&& __in)
: _Inherited(__tag, __a, static_cast<_Tuple_impl<0, _U1, _U2>&&>(__in))
{ }
template<typename _Alloc, typename _U1, typename _U2,
_ImplicitCtor<true, const _U1&, const _U2&> = true>
tuple(allocator_arg_t __tag, const _Alloc& __a,
const pair<_U1, _U2>& __in)
: _Inherited(__tag, __a, __in.first, __in.second) { }
template<typename _Alloc, typename _U1, typename _U2,
_ExplicitCtor<true, const _U1&, const _U2&> = false>
explicit
tuple(allocator_arg_t __tag, const _Alloc& __a,
const pair<_U1, _U2>& __in)
: _Inherited(__tag, __a, __in.first, __in.second) { }
template<typename _Alloc, typename _U1, typename _U2,
_ImplicitCtor<true, _U1, _U2> = true>
tuple(allocator_arg_t __tag, const _Alloc& __a, pair<_U1, _U2>&& __in)
: _Inherited(__tag, __a, std::forward<_U1>(__in.first),
std::forward<_U2>(__in.second)) { }
template<typename _Alloc, typename _U1, typename _U2,
_ExplicitCtor<true, _U1, _U2> = false>
explicit
tuple(allocator_arg_t __tag, const _Alloc& __a, pair<_U1, _U2>&& __in)
: _Inherited(__tag, __a, std::forward<_U1>(__in.first),
std::forward<_U2>(__in.second)) { }
tuple&
operator=(__conditional_t<__assignable<const _T1&, const _T2&>(),
const tuple&,
const __nonesuch&> __in)
noexcept(__nothrow_assignable<const _T1&, const _T2&>())
{
this->_M_assign(__in);
return *this;
}
tuple&
operator=(__conditional_t<__assignable<_T1, _T2>(),
tuple&&,
__nonesuch&&> __in)
noexcept(__nothrow_assignable<_T1, _T2>())
{
this->_M_assign(std::move(__in));
return *this;
}
template<typename _U1, typename _U2>
__enable_if_t<__assignable<const _U1&, const _U2&>(), tuple&>
operator=(const tuple<_U1, _U2>& __in)
noexcept(__nothrow_assignable<const _U1&, const _U2&>())
{
this->_M_assign(__in);
return *this;
}
template<typename _U1, typename _U2>
__enable_if_t<__assignable<_U1, _U2>(), tuple&>
operator=(tuple<_U1, _U2>&& __in)
noexcept(__nothrow_assignable<_U1, _U2>())
{
this->_M_assign(std::move(__in));
return *this;
}
template<typename _U1, typename _U2>
__enable_if_t<__assignable<const _U1&, const _U2&>(), tuple&>
operator=(const pair<_U1, _U2>& __in)
noexcept(__nothrow_assignable<const _U1&, const _U2&>())
{
this->_M_head(*this) = __in.first;
this->_M_tail(*this)._M_head(*this) = __in.second;
return *this;
}
template<typename _U1, typename _U2>
__enable_if_t<__assignable<_U1, _U2>(), tuple&>
operator=(pair<_U1, _U2>&& __in)
noexcept(__nothrow_assignable<_U1, _U2>())
{
this->_M_head(*this) = std::forward<_U1>(__in.first);
this->_M_tail(*this)._M_head(*this) = std::forward<_U2>(__in.second);
return *this;
}
void
swap(tuple& __in)
noexcept(__and_<__is_nothrow_swappable<_T1>,
__is_nothrow_swappable<_T2>>::value)
{ _Inherited::_M_swap(__in); }
};
template<typename... _Elements>
struct tuple_size<tuple<_Elements...>>
: public integral_constant<size_t, sizeof...(_Elements)> { };
template<typename... _Types>
inline constexpr size_t tuple_size_v<tuple<_Types...>>
= sizeof...(_Types);
template<typename... _Types>
inline constexpr size_t tuple_size_v<const tuple<_Types...>>
= sizeof...(_Types);
template<size_t __i, typename... _Types>
struct tuple_element<__i, tuple<_Types...>>
{
static_assert(__i < sizeof...(_Types), "tuple index must be in range");
using type = typename _Nth_type<__i, _Types...>::type;
};
template<size_t __i, typename _Head, typename... _Tail>
constexpr _Head&
__get_helper(_Tuple_impl<__i, _Head, _Tail...>& __t) noexcept
{ return _Tuple_impl<__i, _Head, _Tail...>::_M_head(__t); }
template<size_t __i, typename _Head, typename... _Tail>
constexpr const _Head&
__get_helper(const _Tuple_impl<__i, _Head, _Tail...>& __t) noexcept
{ return _Tuple_impl<__i, _Head, _Tail...>::_M_head(__t); }
template<size_t __i, typename... _Types>
__enable_if_t<(__i >= sizeof...(_Types))>
__get_helper(const tuple<_Types...>&) = delete;
template<size_t __i, typename... _Elements>
constexpr __tuple_element_t<__i, tuple<_Elements...>>&
get(tuple<_Elements...>& __t) noexcept
{ return std::__get_helper<__i>(__t); }
template<size_t __i, typename... _Elements>
constexpr const __tuple_element_t<__i, tuple<_Elements...>>&
get(const tuple<_Elements...>& __t) noexcept
{ return std::__get_helper<__i>(__t); }
template<size_t __i, typename... _Elements>
constexpr __tuple_element_t<__i, tuple<_Elements...>>&&
get(tuple<_Elements...>&& __t) noexcept
{
typedef __tuple_element_t<__i, tuple<_Elements...>> __element_type;
return std::forward<__element_type>(std::__get_helper<__i>(__t));
}
template<size_t __i, typename... _Elements>
constexpr const __tuple_element_t<__i, tuple<_Elements...>>&&
get(const tuple<_Elements...>&& __t) noexcept
{
typedef __tuple_element_t<__i, tuple<_Elements...>> __element_type;
return std::forward<const __element_type>(std::__get_helper<__i>(__t));
}
template<size_t __i, typename... _Elements>
constexpr __enable_if_t<(__i >= sizeof...(_Elements))>
get(const tuple<_Elements...>&) = delete;
template <typename _Tp, typename... _Types>
constexpr _Tp&
get(tuple<_Types...>& __t) noexcept
{
constexpr size_t __idx = __find_uniq_type_in_pack<_Tp, _Types...>();
static_assert(__idx < sizeof...(_Types),
"the type T in std::get<T> must occur exactly once in the tuple");
return std::__get_helper<__idx>(__t);
}
template <typename _Tp, typename... _Types>
constexpr _Tp&&
get(tuple<_Types...>&& __t) noexcept
{
constexpr size_t __idx = __find_uniq_type_in_pack<_Tp, _Types...>();
static_assert(__idx < sizeof...(_Types),
"the type T in std::get<T> must occur exactly once in the tuple");
return std::forward<_Tp>(std::__get_helper<__idx>(__t));
}
template <typename _Tp, typename... _Types>
constexpr const _Tp&
get(const tuple<_Types...>& __t) noexcept
{
constexpr size_t __idx = __find_uniq_type_in_pack<_Tp, _Types...>();
static_assert(__idx < sizeof...(_Types),
"the type T in std::get<T> must occur exactly once in the tuple");
return std::__get_helper<__idx>(__t);
}
template <typename _Tp, typename... _Types>
constexpr const _Tp&&
get(const tuple<_Types...>&& __t) noexcept
{
constexpr size_t __idx = __find_uniq_type_in_pack<_Tp, _Types...>();
static_assert(__idx < sizeof...(_Types),
"the type T in std::get<T> must occur exactly once in the tuple");
return std::forward<const _Tp>(std::__get_helper<__idx>(__t));
}
template<typename _Tp, typename _Up, size_t __i, size_t __size>
struct __tuple_compare
{
static constexpr bool
__eq(const _Tp& __t, const _Up& __u)
{
return bool(std::get<__i>(__t) == std::get<__i>(__u))
&& __tuple_compare<_Tp, _Up, __i + 1, __size>::__eq(__t, __u);
}
static constexpr bool
__less(const _Tp& __t, const _Up& __u)
{
return bool(std::get<__i>(__t) < std::get<__i>(__u))
|| (!bool(std::get<__i>(__u) < std::get<__i>(__t))
&& __tuple_compare<_Tp, _Up, __i + 1, __size>::__less(__t, __u));
}
};
template<typename _Tp, typename _Up, size_t __size>
struct __tuple_compare<_Tp, _Up, __size, __size>
{
static constexpr bool
__eq(const _Tp&, const _Up&) { return true; }
static constexpr bool
__less(const _Tp&, const _Up&) { return false; }
};
template<typename... _TElements, typename... _UElements>
constexpr bool
operator==(const tuple<_TElements...>& __t,
const tuple<_UElements...>& __u)
{
static_assert(sizeof...(_TElements) == sizeof...(_UElements),
"tuple objects can only be compared if they have equal sizes.");
using __compare = __tuple_compare<tuple<_TElements...>,
tuple<_UElements...>,
0, sizeof...(_TElements)>;
return __compare::__eq(__t, __u);
}
template<typename... _TElements, typename... _UElements>
constexpr bool
operator<(const tuple<_TElements...>& __t,
const tuple<_UElements...>& __u)
{
static_assert(sizeof...(_TElements) == sizeof...(_UElements),
"tuple objects can only be compared if they have equal sizes.");
using __compare = __tuple_compare<tuple<_TElements...>,
tuple<_UElements...>,
0, sizeof...(_TElements)>;
return __compare::__less(__t, __u);
}
template<typename... _TElements, typename... _UElements>
constexpr bool
operator!=(const tuple<_TElements...>& __t,
const tuple<_UElements...>& __u)
{ return !(__t == __u); }
template<typename... _TElements, typename... _UElements>
constexpr bool
operator>(const tuple<_TElements...>& __t,
const tuple<_UElements...>& __u)
{ return __u < __t; }
template<typename... _TElements, typename... _UElements>
constexpr bool
operator<=(const tuple<_TElements...>& __t,
const tuple<_UElements...>& __u)
{ return !(__u < __t); }
template<typename... _TElements, typename... _UElements>
constexpr bool
operator>=(const tuple<_TElements...>& __t,
const tuple<_UElements...>& __u)
{ return !(__t < __u); }
template<typename... _Elements>
constexpr tuple<typename __decay_and_strip<_Elements>::__type...>
make_tuple(_Elements&&... __args)
{
typedef tuple<typename __decay_and_strip<_Elements>::__type...>
__result_type;
return __result_type(std::forward<_Elements>(__args)...);
}
template<typename... _Elements>
constexpr tuple<_Elements&&...>
forward_as_tuple(_Elements&&... __args) noexcept
{ return tuple<_Elements&&...>(std::forward<_Elements>(__args)...); }
template<typename _Tp, size_t _Nm> struct array;
template<size_t _Int, typename _Tp, size_t _Nm>
constexpr _Tp&
get(array<_Tp, _Nm>&) noexcept;
template<size_t _Int, typename _Tp, size_t _Nm>
constexpr _Tp&&
get(array<_Tp, _Nm>&&) noexcept;
template<size_t _Int, typename _Tp, size_t _Nm>
constexpr const _Tp&
get(const array<_Tp, _Nm>&) noexcept;
template<size_t _Int, typename _Tp, size_t _Nm>
constexpr const _Tp&&
get(const array<_Tp, _Nm>&&) noexcept;
template<size_t, typename, typename, size_t>
struct __make_tuple_impl;
template<size_t _Idx, typename _Tuple, typename... _Tp, size_t _Nm>
struct __make_tuple_impl<_Idx, tuple<_Tp...>, _Tuple, _Nm>
: __make_tuple_impl<_Idx + 1,
tuple<_Tp..., __tuple_element_t<_Idx, _Tuple>>,
_Tuple, _Nm>
{ };
template<size_t _Nm, typename _Tuple, typename... _Tp>
struct __make_tuple_impl<_Nm, tuple<_Tp...>, _Tuple, _Nm>
{
typedef tuple<_Tp...> __type;
};
template<typename _Tuple>
struct __do_make_tuple
: __make_tuple_impl<0, tuple<>, _Tuple, tuple_size<_Tuple>::value>
{ };
template<typename _Tuple>
struct __make_tuple
: public __do_make_tuple<__remove_cvref_t<_Tuple>>
{ };
template<typename...>
struct __combine_tuples;
template<>
struct __combine_tuples<>
{
typedef tuple<> __type;
};
template<typename... _Ts>
struct __combine_tuples<tuple<_Ts...>>
{
typedef tuple<_Ts...> __type;
};
template<typename... _T1s, typename... _T2s, typename... _Rem>
struct __combine_tuples<tuple<_T1s...>, tuple<_T2s...>, _Rem...>
{
typedef typename __combine_tuples<tuple<_T1s..., _T2s...>,
_Rem...>::__type __type;
};
template<typename... _Tpls>
struct __tuple_cat_result
{
typedef typename __combine_tuples
<typename __make_tuple<_Tpls>::__type...>::__type __type;
};
template<typename...>
struct __make_1st_indices;
template<>
struct __make_1st_indices<>
{
typedef _Index_tuple<> __type;
};
template<typename _Tp, typename... _Tpls>
struct __make_1st_indices<_Tp, _Tpls...>
{
typedef typename _Build_index_tuple<tuple_size<
typename remove_reference<_Tp>::type>::value>::__type __type;
};
template<typename _Ret, typename _Indices, typename... _Tpls>
struct __tuple_concater;
template<typename _Ret, size_t... _Is, typename _Tp, typename... _Tpls>
struct __tuple_concater<_Ret, _Index_tuple<_Is...>, _Tp, _Tpls...>
{
template<typename... _Us>
static constexpr _Ret
_S_do(_Tp&& __tp, _Tpls&&... __tps, _Us&&... __us)
{
typedef typename __make_1st_indices<_Tpls...>::__type __idx;
typedef __tuple_concater<_Ret, __idx, _Tpls...> __next;
return __next::_S_do(std::forward<_Tpls>(__tps)...,
std::forward<_Us>(__us)...,
std::get<_Is>(std::forward<_Tp>(__tp))...);
}
};
template<typename _Ret>
struct __tuple_concater<_Ret, _Index_tuple<>>
{
template<typename... _Us>
static constexpr _Ret
_S_do(_Us&&... __us)
{
return _Ret(std::forward<_Us>(__us)...);
}
};
template<typename... _Tps>
struct __is_tuple_like_impl<tuple<_Tps...>> : true_type
{ };
template<typename... _Tpls, typename = typename
enable_if<__and_<__is_tuple_like<_Tpls>...>::value>::type>
constexpr auto
tuple_cat(_Tpls&&... __tpls)
-> typename __tuple_cat_result<_Tpls...>::__type
{
typedef typename __tuple_cat_result<_Tpls...>::__type __ret;
typedef typename __make_1st_indices<_Tpls...>::__type __idx;
typedef __tuple_concater<__ret, __idx, _Tpls...> __concater;
return __concater::_S_do(std::forward<_Tpls>(__tpls)...);
}
template<typename... _Elements>
constexpr tuple<_Elements&...>
tie(_Elements&... __args) noexcept
{ return tuple<_Elements&...>(__args...); }
template<typename... _Elements>
inline
typename enable_if<__and_<__is_swappable<_Elements>...>::value
>::type
swap(tuple<_Elements...>& __x, tuple<_Elements...>& __y)
noexcept(noexcept(__x.swap(__y)))
{ __x.swap(__y); }
template<typename... _Elements>
typename enable_if<!__and_<__is_swappable<_Elements>...>::value>::type
swap(tuple<_Elements...>&, tuple<_Elements...>&) = delete;
struct _Swallow_assign
{
template<class _Tp>
constexpr const _Swallow_assign&
operator=(const _Tp&) const
{ return *this; }
};
inline constexpr _Swallow_assign ignore{};
template<typename... _Types, typename _Alloc>
struct uses_allocator<tuple<_Types...>, _Alloc> : true_type { };
template<class _T1, class _T2>
template<typename... _Args1, typename... _Args2>
inline
pair<_T1, _T2>::
pair(piecewise_construct_t,
tuple<_Args1...> __first, tuple<_Args2...> __second)
: pair(__first, __second,
typename _Build_index_tuple<sizeof...(_Args1)>::__type(),
typename _Build_index_tuple<sizeof...(_Args2)>::__type())
{ }
template<class _T1, class _T2>
template<typename... _Args1, size_t... _Indexes1,
typename... _Args2, size_t... _Indexes2>
inline
pair<_T1, _T2>::
pair(tuple<_Args1...>& __tuple1, tuple<_Args2...>& __tuple2,
_Index_tuple<_Indexes1...>, _Index_tuple<_Indexes2...>)
: first(std::forward<_Args1>(std::get<_Indexes1>(__tuple1))...),
second(std::forward<_Args2>(std::get<_Indexes2>(__tuple2))...)
{ }
template<template<typename...> class _Trait, typename _Tp, typename _Tuple>
inline constexpr bool __unpack_std_tuple = false;
template<template<typename...> class _Trait, typename _Tp, typename... _Up>
inline constexpr bool __unpack_std_tuple<_Trait, _Tp, tuple<_Up...>>
= _Trait<_Tp, _Up...>::value;
template<template<typename...> class _Trait, typename _Tp, typename... _Up>
inline constexpr bool __unpack_std_tuple<_Trait, _Tp, tuple<_Up...>&>
= _Trait<_Tp, _Up&...>::value;
template<template<typename...> class _Trait, typename _Tp, typename... _Up>
inline constexpr bool __unpack_std_tuple<_Trait, _Tp, const tuple<_Up...>>
= _Trait<_Tp, const _Up...>::value;
template<template<typename...> class _Trait, typename _Tp, typename... _Up>
inline constexpr bool __unpack_std_tuple<_Trait, _Tp, const tuple<_Up...>&>
= _Trait<_Tp, const _Up&...>::value;
template <typename _Fn, typename _Tuple, size_t... _Idx>
constexpr decltype(auto)
__apply_impl(_Fn&& __f, _Tuple&& __t, index_sequence<_Idx...>)
{
return std::__invoke(std::forward<_Fn>(__f),
std::get<_Idx>(std::forward<_Tuple>(__t))...);
}
template <typename _Fn, typename _Tuple>
constexpr decltype(auto)
apply(_Fn&& __f, _Tuple&& __t)
noexcept(__unpack_std_tuple<is_nothrow_invocable, _Fn, _Tuple>)
{
using _Indices
= make_index_sequence<tuple_size_v<remove_reference_t<_Tuple>>>;
return std::__apply_impl(std::forward<_Fn>(__f),
std::forward<_Tuple>(__t),
_Indices{});
}
template <typename _Tp, typename _Tuple, size_t... _Idx>
constexpr _Tp
__make_from_tuple_impl(_Tuple&& __t, index_sequence<_Idx...>)
{ return _Tp(std::get<_Idx>(std::forward<_Tuple>(__t))...); }
template <typename _Tp, typename _Tuple>
constexpr _Tp
make_from_tuple(_Tuple&& __t)
noexcept(__unpack_std_tuple<is_nothrow_constructible, _Tp, _Tuple>)
{
constexpr size_t __n = tuple_size_v<remove_reference_t<_Tuple>>;
if constexpr (__n == 1)
{
using _Elt = decltype(std::get<0>(std::declval<_Tuple>()));
static_assert(!__reference_constructs_from_temporary(_Tp, _Elt));
}
return __make_from_tuple_impl<_Tp>(std::forward<_Tuple>(__t),
make_index_sequence<__n>{});
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace pmr
{
class memory_resource
{
static constexpr size_t _S_max_align = alignof(max_align_t);
public:
memory_resource() = default;
memory_resource(const memory_resource&) = default;
virtual ~memory_resource();
memory_resource& operator=(const memory_resource&) = default;
[[nodiscard]]
void*
allocate(size_t __bytes, size_t __alignment = _S_max_align)
__attribute__((__returns_nonnull__,__alloc_size__(2),__alloc_align__(3)))
{ return ::operator new(__bytes, do_allocate(__bytes, __alignment)); }
void
deallocate(void* __p, size_t __bytes, size_t __alignment = _S_max_align)
__attribute__((__nonnull__))
{ return do_deallocate(__p, __bytes, __alignment); }
[[nodiscard]]
bool
is_equal(const memory_resource& __other) const noexcept
{ return do_is_equal(__other); }
private:
virtual void*
do_allocate(size_t __bytes, size_t __alignment) = 0;
virtual void
do_deallocate(void* __p, size_t __bytes, size_t __alignment) = 0;
virtual bool
do_is_equal(const memory_resource& __other) const noexcept = 0;
};
[[nodiscard]]
inline bool
operator==(const memory_resource& __a, const memory_resource& __b) noexcept
{ return &__a == &__b || __a.is_equal(__b); }
[[nodiscard]]
inline bool
operator!=(const memory_resource& __a, const memory_resource& __b) noexcept
{ return !(__a == __b); }
template<typename _Tp>
class polymorphic_allocator
{
template<typename _Up>
struct __not_pair { using type = void; };
template<typename _Up1, typename _Up2>
struct __not_pair<pair<_Up1, _Up2>> { };
public:
using value_type = _Tp;
polymorphic_allocator() noexcept
{
extern memory_resource* get_default_resource() noexcept
__attribute__((__returns_nonnull__));
_M_resource = get_default_resource();
}
polymorphic_allocator(memory_resource* __r) noexcept
__attribute__((__nonnull__))
: _M_resource(__r)
{ ; }
polymorphic_allocator(const polymorphic_allocator& __other) = default;
template<typename _Up>
polymorphic_allocator(const polymorphic_allocator<_Up>& __x) noexcept
: _M_resource(__x.resource())
{ }
polymorphic_allocator&
operator=(const polymorphic_allocator&) = delete;
[[nodiscard]]
_Tp*
allocate(size_t __n)
__attribute__((__returns_nonnull__))
{
if ((__gnu_cxx::__int_traits<size_t>::__max / sizeof(_Tp)) < __n)
std::__throw_bad_array_new_length();
return static_cast<_Tp*>(_M_resource->allocate(__n * sizeof(_Tp),
alignof(_Tp)));
}
void
deallocate(_Tp* __p, size_t __n) noexcept
__attribute__((__nonnull__))
{ _M_resource->deallocate(__p, __n * sizeof(_Tp), alignof(_Tp)); }
template<typename _Tp1, typename... _Args>
__attribute__((__nonnull__))
typename __not_pair<_Tp1>::type
construct(_Tp1* __p, _Args&&... __args)
{
using __use_tag
= std::__uses_alloc_t<_Tp1, polymorphic_allocator, _Args...>;
if constexpr (is_base_of_v<__uses_alloc0, __use_tag>)
::new(__p) _Tp1(std::forward<_Args>(__args)...);
else if constexpr (is_base_of_v<__uses_alloc1_, __use_tag>)
::new(__p) _Tp1(allocator_arg, *this,
std::forward<_Args>(__args)...);
else
::new(__p) _Tp1(std::forward<_Args>(__args)..., *this);
}
template<typename _Tp1, typename _Tp2,
typename... _Args1, typename... _Args2>
__attribute__((__nonnull__))
void
construct(pair<_Tp1, _Tp2>* __p, piecewise_construct_t,
tuple<_Args1...> __x, tuple<_Args2...> __y)
{
auto __x_tag =
__use_alloc<_Tp1, polymorphic_allocator, _Args1...>(*this);
auto __y_tag =
__use_alloc<_Tp2, polymorphic_allocator, _Args2...>(*this);
index_sequence_for<_Args1...> __x_i;
index_sequence_for<_Args2...> __y_i;
::new(__p) pair<_Tp1, _Tp2>(piecewise_construct,
_S_construct_p(__x_tag, __x_i, __x),
_S_construct_p(__y_tag, __y_i, __y));
}
template<typename _Tp1, typename _Tp2>
__attribute__((__nonnull__))
void
construct(pair<_Tp1, _Tp2>* __p)
{ this->construct(__p, piecewise_construct, tuple<>(), tuple<>()); }
template<typename _Tp1, typename _Tp2, typename _Up, typename _Vp>
__attribute__((__nonnull__))
void
construct(pair<_Tp1, _Tp2>* __p, _Up&& __x, _Vp&& __y)
{
this->construct(__p, piecewise_construct,
std::forward_as_tuple(std::forward<_Up>(__x)),
std::forward_as_tuple(std::forward<_Vp>(__y)));
}
template <typename _Tp1, typename _Tp2, typename _Up, typename _Vp>
__attribute__((__nonnull__))
void
construct(pair<_Tp1, _Tp2>* __p, const std::pair<_Up, _Vp>& __pr)
{
this->construct(__p, piecewise_construct,
std::forward_as_tuple(__pr.first),
std::forward_as_tuple(__pr.second));
}
template<typename _Tp1, typename _Tp2, typename _Up, typename _Vp>
__attribute__((__nonnull__))
void
construct(pair<_Tp1, _Tp2>* __p, pair<_Up, _Vp>&& __pr)
{
this->construct(__p, piecewise_construct,
std::forward_as_tuple(std::forward<_Up>(__pr.first)),
std::forward_as_tuple(std::forward<_Vp>(__pr.second)));
}
template<typename _Up>
__attribute__((__nonnull__))
void
destroy(_Up* __p)
{ __p->~_Up(); }
polymorphic_allocator
select_on_container_copy_construction() const noexcept
{ return polymorphic_allocator(); }
memory_resource*
resource() const noexcept
__attribute__((__returns_nonnull__))
{ return _M_resource; }
[[nodiscard]]
friend bool
operator==(const polymorphic_allocator& __a,
const polymorphic_allocator& __b) noexcept
{ return *__a.resource() == *__b.resource(); }
[[nodiscard]]
friend bool
operator!=(const polymorphic_allocator& __a,
const polymorphic_allocator& __b) noexcept
{ return !(__a == __b); }
private:
using __uses_alloc1_ = __uses_alloc1<polymorphic_allocator>;
using __uses_alloc2_ = __uses_alloc2<polymorphic_allocator>;
template<typename _Ind, typename... _Args>
static tuple<_Args&&...>
_S_construct_p(__uses_alloc0, _Ind, tuple<_Args...>& __t)
{ return std::move(__t); }
template<size_t... _Ind, typename... _Args>
static tuple<allocator_arg_t, polymorphic_allocator, _Args&&...>
_S_construct_p(__uses_alloc1_ __ua, index_sequence<_Ind...>,
tuple<_Args...>& __t)
{
return {
allocator_arg, *__ua._M_a, std::get<_Ind>(std::move(__t))...
};
}
template<size_t... _Ind, typename... _Args>
static tuple<_Args&&..., polymorphic_allocator>
_S_construct_p(__uses_alloc2_ __ua, index_sequence<_Ind...>,
tuple<_Args...>& __t)
{ return { std::get<_Ind>(std::move(__t))..., *__ua._M_a }; }
memory_resource* _M_resource;
};
template<typename _Tp1, typename _Tp2>
[[nodiscard]]
inline bool
operator==(const polymorphic_allocator<_Tp1>& __a,
const polymorphic_allocator<_Tp2>& __b) noexcept
{ return *__a.resource() == *__b.resource(); }
template<typename _Tp1, typename _Tp2>
[[nodiscard]]
inline bool
operator!=(const polymorphic_allocator<_Tp1>& __a,
const polymorphic_allocator<_Tp2>& __b) noexcept
{ return !(__a == __b); }
}
template<typename _Alloc> struct allocator_traits;
template<typename _Tp>
struct allocator_traits<pmr::polymorphic_allocator<_Tp>>
{
using allocator_type = pmr::polymorphic_allocator<_Tp>;
using value_type = _Tp;
using pointer = _Tp*;
using const_pointer = const _Tp*;
using void_pointer = void*;
using const_void_pointer = const void*;
using difference_type = std::ptrdiff_t;
using size_type = std::size_t;
using propagate_on_container_copy_assignment = false_type;
using propagate_on_container_move_assignment = false_type;
using propagate_on_container_swap = false_type;
static allocator_type
select_on_container_copy_construction(const allocator_type&) noexcept
{ return allocator_type(); }
using is_always_equal = false_type;
template<typename _Up>
using rebind_alloc = pmr::polymorphic_allocator<_Up>;
template<typename _Up>
using rebind_traits = allocator_traits<pmr::polymorphic_allocator<_Up>>;
[[nodiscard]] static pointer
allocate(allocator_type& __a, size_type __n)
{ return __a.allocate(__n); }
[[nodiscard]] static pointer
allocate(allocator_type& __a, size_type __n, const_void_pointer)
{ return __a.allocate(__n); }
static void
deallocate(allocator_type& __a, pointer __p, size_type __n)
{ __a.deallocate(__p, __n); }
template<typename _Up, typename... _Args>
static void
construct(allocator_type& __a, _Up* __p, _Args&&... __args)
{ __a.construct(__p, std::forward<_Args>(__args)...); }
template<typename _Up>
static void
destroy(allocator_type&, _Up* __p)
noexcept(is_nothrow_destructible<_Up>::value)
{ __p->~_Up(); }
static size_type
max_size(const allocator_type&) noexcept
{ return size_t(-1) / sizeof(value_type); }
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace pmr {
template<typename _Tp>
using vector = std::vector<_Tp, polymorphic_allocator<_Tp>>;
}
}
namespace hwy {
int64_t SupportedTargets();
void DisableTargets(int64_t disabled_targets);
void SetSupportedTargetsForTest(int64_t targets);
inline __attribute__((always_inline)) std::vector<int64_t> SupportedAndGeneratedTargets() {
std::vector<int64_t> ret;
for (int64_t targets = SupportedTargets() & ((((1LL << 61) | (1LL << 14) | (1LL << 12) | (1LL << 11) | (1LL << 9) | (1LL << 8) | 0 | (1LL << 6)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0)))) & (((((((1LL << 61) | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0)))) & -((((1LL << 61) | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0))))) - 1LL) | (((((1LL << 61) | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0)))) & -((((1LL << 61) | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0))))))); targets != 0;
targets = targets & (targets - 1)) {
int64_t current_target = targets & ~(targets - 1);
ret.push_back(current_target);
}
return ret;
}
static inline __attribute__((unused)) const char* TargetName(int64_t target) {
switch (target) {
case (1LL << 14):
return "SSE2";
case (1LL << 12):
return "SSSE3";
case (1LL << 11):
return "SSE4";
case (1LL << 9):
return "AVX2";
case (1LL << 8):
return "AVX3";
case (1LL << 7):
return "AVX3_DL";
case (1LL << 6):
return "AVX3_ZEN4";
case (1LL << 61):
return "EMU128";
case (1LL << 62):
return "SCALAR";
default:
return "Unknown";
}
}
struct ChosenTarget {
public:
void Update(int64_t targets) {
StoreMask(((((targets) >> (14 + 1 - 15)) & ((1LL << 15) - 1)) << 1) | (1LL << (15 + 1)));
}
void DeInit() { StoreMask(1); }
bool IsInitialized() const { return LoadMask() != 1; }
size_t inline __attribute__((always_inline)) GetIndex() const {
return hwy::Num0BitsBelowLS1Bit_Nonzero64(
static_cast<uint64_t>(LoadMask() & (((((((((1LL << 61) | (1LL << 14) | (1LL << 12) | (1LL << 11) | (1LL << 9) | (1LL << 8) | 0 | (1LL << 6)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0)))) & (((((((1LL << 61) | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0)))) & -((((1LL << 61) | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0))))) - 1LL) | (((((1LL << 61) | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0)))) & -((((1LL << 61) | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0)))))))) >> (14 + 1 - 15)) & ((1LL << 15) - 1)) << 1) | (1LL << (15 + 1)) | 1LL)));
}
private:
int64_t LoadMask() const { return mask_.load(); }
void StoreMask(int64_t mask) { mask_.store(mask); }
std::atomic<int64_t> mask_{1};
};
ChosenTarget& GetChosenTarget();
}
namespace hwy {
template <typename RetType, typename... Args>
struct FunctionCache {
public:
typedef RetType(FunctionType)(Args...);
template <FunctionType* const table[]>
static RetType ChooseAndCall(Args... args) {
ChosenTarget& chosen_target = GetChosenTarget();
chosen_target.Update(SupportedTargets());
return (table[chosen_target.GetIndex()])(args...);
}
};
template <typename RetType, typename... Args>
FunctionCache<RetType, Args...> DeduceFunctionCache(RetType (*)(Args...)) {
return FunctionCache<RetType, Args...>();
}
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wuninitialized"
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#pragma GCC push_options
#pragma GCC target("mmx")
typedef int __m64 __attribute__ ((__vector_size__ (8), __may_alias__));
typedef int __m32 __attribute__ ((__vector_size__ (4), __may_alias__));
typedef short __m16 __attribute__ ((__vector_size__ (2), __may_alias__));
typedef int __m64_u __attribute__ ((__vector_size__ (8), __may_alias__, __aligned__ (1)));
typedef int __m32_u __attribute__ ((__vector_size__ (4),
__may_alias__, __aligned__ (1)));
typedef short __m16_u __attribute__ ((__vector_size__ (2),
__may_alias__, __aligned__ (1)));
typedef int __v2si __attribute__ ((__vector_size__ (8)));
typedef short __v4hi __attribute__ ((__vector_size__ (8)));
typedef char __v8qi __attribute__ ((__vector_size__ (8)));
typedef long long __v1di __attribute__ ((__vector_size__ (8)));
typedef float __v2sf __attribute__ ((__vector_size__ (8)));
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_empty (void)
{
__builtin_ia32_emms ();
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_empty (void)
{
_mm_empty ();
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtsi32_si64 (int __i)
{
return (__m64) __builtin_ia32_vec_init_v2si (__i, 0);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_from_int (int __i)
{
return _mm_cvtsi32_si64 (__i);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtsi64_si32 (__m64 __i)
{
return __builtin_ia32_vec_ext_v2si ((__v2si)__i, 0);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_to_int (__m64 __i)
{
return _mm_cvtsi64_si32 (__i);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_packs_pi16 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_packsswb ((__v4hi)__m1, (__v4hi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_packsswb (__m64 __m1, __m64 __m2)
{
return _mm_packs_pi16 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_packs_pi32 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_packssdw ((__v2si)__m1, (__v2si)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_packssdw (__m64 __m1, __m64 __m2)
{
return _mm_packs_pi32 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_packs_pu16 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_packuswb ((__v4hi)__m1, (__v4hi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_packuswb (__m64 __m1, __m64 __m2)
{
return _mm_packs_pu16 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpackhi_pi8 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_punpckhbw ((__v8qi)__m1, (__v8qi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_punpckhbw (__m64 __m1, __m64 __m2)
{
return _mm_unpackhi_pi8 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpackhi_pi16 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_punpckhwd ((__v4hi)__m1, (__v4hi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_punpckhwd (__m64 __m1, __m64 __m2)
{
return _mm_unpackhi_pi16 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpackhi_pi32 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_punpckhdq ((__v2si)__m1, (__v2si)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_punpckhdq (__m64 __m1, __m64 __m2)
{
return _mm_unpackhi_pi32 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpacklo_pi8 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_punpcklbw ((__v8qi)__m1, (__v8qi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_punpcklbw (__m64 __m1, __m64 __m2)
{
return _mm_unpacklo_pi8 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpacklo_pi16 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_punpcklwd ((__v4hi)__m1, (__v4hi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_punpcklwd (__m64 __m1, __m64 __m2)
{
return _mm_unpacklo_pi16 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpacklo_pi32 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_punpckldq ((__v2si)__m1, (__v2si)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_punpckldq (__m64 __m1, __m64 __m2)
{
return _mm_unpacklo_pi32 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_add_pi8 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_paddb ((__v8qi)__m1, (__v8qi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_paddb (__m64 __m1, __m64 __m2)
{
return _mm_add_pi8 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_add_pi16 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_paddw ((__v4hi)__m1, (__v4hi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_paddw (__m64 __m1, __m64 __m2)
{
return _mm_add_pi16 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_add_pi32 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_paddd ((__v2si)__m1, (__v2si)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_paddd (__m64 __m1, __m64 __m2)
{
return _mm_add_pi32 (__m1, __m2);
}
#pragma GCC push_options
#pragma GCC target("sse2,mmx")
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_add_si64 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_paddq ((__v1di)__m1, (__v1di)__m2);
}
#pragma GCC pop_options
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_adds_pi8 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_paddsb ((__v8qi)__m1, (__v8qi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_paddsb (__m64 __m1, __m64 __m2)
{
return _mm_adds_pi8 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_adds_pi16 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_paddsw ((__v4hi)__m1, (__v4hi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_paddsw (__m64 __m1, __m64 __m2)
{
return _mm_adds_pi16 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_adds_pu8 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_paddusb ((__v8qi)__m1, (__v8qi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_paddusb (__m64 __m1, __m64 __m2)
{
return _mm_adds_pu8 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_adds_pu16 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_paddusw ((__v4hi)__m1, (__v4hi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_paddusw (__m64 __m1, __m64 __m2)
{
return _mm_adds_pu16 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sub_pi8 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_psubb ((__v8qi)__m1, (__v8qi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psubb (__m64 __m1, __m64 __m2)
{
return _mm_sub_pi8 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sub_pi16 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_psubw ((__v4hi)__m1, (__v4hi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psubw (__m64 __m1, __m64 __m2)
{
return _mm_sub_pi16 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sub_pi32 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_psubd ((__v2si)__m1, (__v2si)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psubd (__m64 __m1, __m64 __m2)
{
return _mm_sub_pi32 (__m1, __m2);
}
#pragma GCC push_options
#pragma GCC target("sse2,mmx")
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sub_si64 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_psubq ((__v1di)__m1, (__v1di)__m2);
}
#pragma GCC pop_options
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_subs_pi8 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_psubsb ((__v8qi)__m1, (__v8qi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psubsb (__m64 __m1, __m64 __m2)
{
return _mm_subs_pi8 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_subs_pi16 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_psubsw ((__v4hi)__m1, (__v4hi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psubsw (__m64 __m1, __m64 __m2)
{
return _mm_subs_pi16 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_subs_pu8 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_psubusb ((__v8qi)__m1, (__v8qi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psubusb (__m64 __m1, __m64 __m2)
{
return _mm_subs_pu8 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_subs_pu16 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_psubusw ((__v4hi)__m1, (__v4hi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psubusw (__m64 __m1, __m64 __m2)
{
return _mm_subs_pu16 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_madd_pi16 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_pmaddwd ((__v4hi)__m1, (__v4hi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pmaddwd (__m64 __m1, __m64 __m2)
{
return _mm_madd_pi16 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mulhi_pi16 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_pmulhw ((__v4hi)__m1, (__v4hi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pmulhw (__m64 __m1, __m64 __m2)
{
return _mm_mulhi_pi16 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mullo_pi16 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_pmullw ((__v4hi)__m1, (__v4hi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pmullw (__m64 __m1, __m64 __m2)
{
return _mm_mullo_pi16 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sll_pi16 (__m64 __m, __m64 __count)
{
return (__m64) __builtin_ia32_psllw ((__v4hi)__m, (__v4hi)__count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psllw (__m64 __m, __m64 __count)
{
return _mm_sll_pi16 (__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_slli_pi16 (__m64 __m, int __count)
{
return (__m64) __builtin_ia32_psllwi ((__v4hi)__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psllwi (__m64 __m, int __count)
{
return _mm_slli_pi16 (__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sll_pi32 (__m64 __m, __m64 __count)
{
return (__m64) __builtin_ia32_pslld ((__v2si)__m, (__v2si)__count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pslld (__m64 __m, __m64 __count)
{
return _mm_sll_pi32 (__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_slli_pi32 (__m64 __m, int __count)
{
return (__m64) __builtin_ia32_pslldi ((__v2si)__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pslldi (__m64 __m, int __count)
{
return _mm_slli_pi32 (__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sll_si64 (__m64 __m, __m64 __count)
{
return (__m64) __builtin_ia32_psllq ((__v1di)__m, (__v1di)__count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psllq (__m64 __m, __m64 __count)
{
return _mm_sll_si64 (__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_slli_si64 (__m64 __m, int __count)
{
return (__m64) __builtin_ia32_psllqi ((__v1di)__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psllqi (__m64 __m, int __count)
{
return _mm_slli_si64 (__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sra_pi16 (__m64 __m, __m64 __count)
{
return (__m64) __builtin_ia32_psraw ((__v4hi)__m, (__v4hi)__count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psraw (__m64 __m, __m64 __count)
{
return _mm_sra_pi16 (__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_srai_pi16 (__m64 __m, int __count)
{
return (__m64) __builtin_ia32_psrawi ((__v4hi)__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psrawi (__m64 __m, int __count)
{
return _mm_srai_pi16 (__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sra_pi32 (__m64 __m, __m64 __count)
{
return (__m64) __builtin_ia32_psrad ((__v2si)__m, (__v2si)__count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psrad (__m64 __m, __m64 __count)
{
return _mm_sra_pi32 (__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_srai_pi32 (__m64 __m, int __count)
{
return (__m64) __builtin_ia32_psradi ((__v2si)__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psradi (__m64 __m, int __count)
{
return _mm_srai_pi32 (__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_srl_pi16 (__m64 __m, __m64 __count)
{
return (__m64) __builtin_ia32_psrlw ((__v4hi)__m, (__v4hi)__count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psrlw (__m64 __m, __m64 __count)
{
return _mm_srl_pi16 (__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_srli_pi16 (__m64 __m, int __count)
{
return (__m64) __builtin_ia32_psrlwi ((__v4hi)__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psrlwi (__m64 __m, int __count)
{
return _mm_srli_pi16 (__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_srl_pi32 (__m64 __m, __m64 __count)
{
return (__m64) __builtin_ia32_psrld ((__v2si)__m, (__v2si)__count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psrld (__m64 __m, __m64 __count)
{
return _mm_srl_pi32 (__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_srli_pi32 (__m64 __m, int __count)
{
return (__m64) __builtin_ia32_psrldi ((__v2si)__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psrldi (__m64 __m, int __count)
{
return _mm_srli_pi32 (__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_srl_si64 (__m64 __m, __m64 __count)
{
return (__m64) __builtin_ia32_psrlq ((__v1di)__m, (__v1di)__count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psrlq (__m64 __m, __m64 __count)
{
return _mm_srl_si64 (__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_srli_si64 (__m64 __m, int __count)
{
return (__m64) __builtin_ia32_psrlqi ((__v1di)__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psrlqi (__m64 __m, int __count)
{
return _mm_srli_si64 (__m, __count);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_and_si64 (__m64 __m1, __m64 __m2)
{
return __builtin_ia32_pand (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pand (__m64 __m1, __m64 __m2)
{
return _mm_and_si64 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_andnot_si64 (__m64 __m1, __m64 __m2)
{
return __builtin_ia32_pandn (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pandn (__m64 __m1, __m64 __m2)
{
return _mm_andnot_si64 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_or_si64 (__m64 __m1, __m64 __m2)
{
return __builtin_ia32_por (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_por (__m64 __m1, __m64 __m2)
{
return _mm_or_si64 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_xor_si64 (__m64 __m1, __m64 __m2)
{
return __builtin_ia32_pxor (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pxor (__m64 __m1, __m64 __m2)
{
return _mm_xor_si64 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpeq_pi8 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_pcmpeqb ((__v8qi)__m1, (__v8qi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pcmpeqb (__m64 __m1, __m64 __m2)
{
return _mm_cmpeq_pi8 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpgt_pi8 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_pcmpgtb ((__v8qi)__m1, (__v8qi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pcmpgtb (__m64 __m1, __m64 __m2)
{
return _mm_cmpgt_pi8 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpeq_pi16 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_pcmpeqw ((__v4hi)__m1, (__v4hi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pcmpeqw (__m64 __m1, __m64 __m2)
{
return _mm_cmpeq_pi16 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpgt_pi16 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_pcmpgtw ((__v4hi)__m1, (__v4hi)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pcmpgtw (__m64 __m1, __m64 __m2)
{
return _mm_cmpgt_pi16 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpeq_pi32 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_pcmpeqd ((__v2si)__m1, (__v2si)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pcmpeqd (__m64 __m1, __m64 __m2)
{
return _mm_cmpeq_pi32 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpgt_pi32 (__m64 __m1, __m64 __m2)
{
return (__m64) __builtin_ia32_pcmpgtd ((__v2si)__m1, (__v2si)__m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pcmpgtd (__m64 __m1, __m64 __m2)
{
return _mm_cmpgt_pi32 (__m1, __m2);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_setzero_si64 (void)
{
return (__m64)0LL;
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set_pi32 (int __i1, int __i0)
{
return (__m64) __builtin_ia32_vec_init_v2si (__i0, __i1);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set_pi16 (short __w3, short __w2, short __w1, short __w0)
{
return (__m64) __builtin_ia32_vec_init_v4hi (__w0, __w1, __w2, __w3);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set_pi8 (char __b7, char __b6, char __b5, char __b4,
char __b3, char __b2, char __b1, char __b0)
{
return (__m64) __builtin_ia32_vec_init_v8qi (__b0, __b1, __b2, __b3,
__b4, __b5, __b6, __b7);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_setr_pi32 (int __i0, int __i1)
{
return _mm_set_pi32 (__i1, __i0);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_setr_pi16 (short __w0, short __w1, short __w2, short __w3)
{
return _mm_set_pi16 (__w3, __w2, __w1, __w0);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_setr_pi8 (char __b0, char __b1, char __b2, char __b3,
char __b4, char __b5, char __b6, char __b7)
{
return _mm_set_pi8 (__b7, __b6, __b5, __b4, __b3, __b2, __b1, __b0);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set1_pi32 (int __i)
{
return _mm_set_pi32 (__i, __i);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set1_pi16 (short __w)
{
return _mm_set_pi16 (__w, __w, __w, __w);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set1_pi8 (char __b)
{
return _mm_set_pi8 (__b, __b, __b, __b, __b, __b, __b, __b);
}
#pragma GCC pop_options
extern "C++"
{
namespace std __attribute__ ((__visibility__ ("default")))
{
using ::div_t;
using ::ldiv_t;
using ::abort;
using ::aligned_alloc;
using ::atexit;
using ::at_quick_exit;
using ::atof;
using ::atoi;
using ::atol;
using ::bsearch;
using ::calloc;
using ::div;
using ::exit;
using ::free;
using ::getenv;
using ::labs;
using ::ldiv;
using ::malloc;
using ::mblen;
using ::mbstowcs;
using ::mbtowc;
using ::qsort;
using ::quick_exit;
using ::rand;
using ::realloc;
using ::srand;
using ::strtod;
using ::strtol;
using ::strtoul;
using ::system;
using ::wcstombs;
using ::wctomb;
inline ldiv_t
div(long __i, long __j) noexcept { return ldiv(__i, __j); }
}
namespace __gnu_cxx __attribute__ ((__visibility__ ("default")))
{
using ::lldiv_t;
using ::_Exit;
using ::llabs;
inline lldiv_t
div(long long __n, long long __d)
{ lldiv_t __q; __q.quot = __n / __d; __q.rem = __n % __d; return __q; }
using ::lldiv;
using ::atoll;
using ::strtoll;
using ::strtoull;
using ::strtof;
using ::strtold;
}
namespace std
{
using ::__gnu_cxx::lldiv_t;
using ::__gnu_cxx::_Exit;
using ::__gnu_cxx::llabs;
using ::__gnu_cxx::div;
using ::__gnu_cxx::lldiv;
using ::__gnu_cxx::atoll;
using ::__gnu_cxx::strtof;
using ::__gnu_cxx::strtoll;
using ::__gnu_cxx::strtoull;
using ::__gnu_cxx::strtold;
}
}
using std::abort;
using std::atexit;
using std::exit;
using std::at_quick_exit;
using std::quick_exit;
using std::_Exit;
using std::div_t;
using std::ldiv_t;
using std::abs;
using std::atof;
using std::atoi;
using std::atol;
using std::bsearch;
using std::calloc;
using std::div;
using std::free;
using std::getenv;
using std::labs;
using std::ldiv;
using std::malloc;
using std::mblen;
using std::mbstowcs;
using std::mbtowc;
using std::qsort;
using std::rand;
using std::realloc;
using std::srand;
using std::strtod;
using std::strtol;
using std::strtoul;
using std::system;
using std::wcstombs;
using std::wctomb;
extern "C" int posix_memalign (void **, size_t, size_t) throw ();
static __inline void *
_mm_malloc (size_t __size, size_t __alignment)
{
void *__ptr;
if (__alignment == 1)
return malloc (__size);
if (__alignment == 2 || (sizeof (void *) == 8 && __alignment == 4))
__alignment = sizeof (void *);
if (posix_memalign (&__ptr, __alignment, __size) == 0)
return __ptr;
else
return __null;
}
static __inline void
_mm_free (void *__ptr)
{
free (__ptr);
}
enum _mm_hint
{
_MM_HINT_IT0 = 19,
_MM_HINT_IT1 = 18,
_MM_HINT_ET0 = 7,
_MM_HINT_ET1 = 6,
_MM_HINT_T0 = 3,
_MM_HINT_T1 = 2,
_MM_HINT_T2 = 1,
_MM_HINT_NTA = 0
};
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_prefetch (const void *__P, enum _mm_hint __I)
{
__builtin_ia32_prefetch (__P, (__I & 0x4) >> 2,
__I & 0x3, (__I & 0x10) >> 4);
}
#pragma GCC push_options
#pragma GCC target("sse")
typedef float __m128 __attribute__ ((__vector_size__ (16), __may_alias__));
typedef float __m128_u __attribute__ ((__vector_size__ (16), __may_alias__, __aligned__ (1)));
typedef float __v4sf __attribute__ ((__vector_size__ (16)));
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_undefined_ps (void)
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Winit-self"
__m128 __Y = __Y;
#pragma GCC diagnostic pop
return __Y;
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_setzero_ps (void)
{
return __extension__ (__m128){ 0.0f, 0.0f, 0.0f, 0.0f };
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_add_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_addss ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sub_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_subss ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mul_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_mulss ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_div_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_divss ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sqrt_ss (__m128 __A)
{
return (__m128) __builtin_ia32_sqrtss ((__v4sf)__A);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_rcp_ss (__m128 __A)
{
return (__m128) __builtin_ia32_rcpss ((__v4sf)__A);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_rsqrt_ss (__m128 __A)
{
return (__m128) __builtin_ia32_rsqrtss ((__v4sf)__A);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_min_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_minss ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_max_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_maxss ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_add_ps (__m128 __A, __m128 __B)
{
return (__m128) ((__v4sf)__A + (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sub_ps (__m128 __A, __m128 __B)
{
return (__m128) ((__v4sf)__A - (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mul_ps (__m128 __A, __m128 __B)
{
return (__m128) ((__v4sf)__A * (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_div_ps (__m128 __A, __m128 __B)
{
return (__m128) ((__v4sf)__A / (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sqrt_ps (__m128 __A)
{
return (__m128) __builtin_ia32_sqrtps ((__v4sf)__A);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_rcp_ps (__m128 __A)
{
return (__m128) __builtin_ia32_rcpps ((__v4sf)__A);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_rsqrt_ps (__m128 __A)
{
return (__m128) __builtin_ia32_rsqrtps ((__v4sf)__A);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_min_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_minps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_max_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_maxps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_and_ps (__m128 __A, __m128 __B)
{
return __builtin_ia32_andps (__A, __B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_andnot_ps (__m128 __A, __m128 __B)
{
return __builtin_ia32_andnps (__A, __B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_or_ps (__m128 __A, __m128 __B)
{
return __builtin_ia32_orps (__A, __B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_xor_ps (__m128 __A, __m128 __B)
{
return __builtin_ia32_xorps (__A, __B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpeq_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpeqss ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmplt_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpltss ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmple_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpless ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpgt_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_movss ((__v4sf) __A,
(__v4sf)
__builtin_ia32_cmpltss ((__v4sf) __B,
(__v4sf)
__A));
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpge_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_movss ((__v4sf) __A,
(__v4sf)
__builtin_ia32_cmpless ((__v4sf) __B,
(__v4sf)
__A));
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpneq_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpneqss ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnlt_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpnltss ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnle_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpnless ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpngt_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_movss ((__v4sf) __A,
(__v4sf)
__builtin_ia32_cmpnltss ((__v4sf) __B,
(__v4sf)
__A));
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnge_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_movss ((__v4sf) __A,
(__v4sf)
__builtin_ia32_cmpnless ((__v4sf) __B,
(__v4sf)
__A));
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpord_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpordss ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpunord_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpunordss ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpeq_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpeqps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmplt_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpltps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmple_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpleps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpgt_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpgtps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpge_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpgeps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpneq_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpneqps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnlt_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpnltps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnle_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpnleps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpngt_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpngtps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnge_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpngeps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpord_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpordps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpunord_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_cmpunordps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comieq_ss (__m128 __A, __m128 __B)
{
return __builtin_ia32_comieq ((__v4sf)__A, (__v4sf)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comilt_ss (__m128 __A, __m128 __B)
{
return __builtin_ia32_comilt ((__v4sf)__A, (__v4sf)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comile_ss (__m128 __A, __m128 __B)
{
return __builtin_ia32_comile ((__v4sf)__A, (__v4sf)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comigt_ss (__m128 __A, __m128 __B)
{
return __builtin_ia32_comigt ((__v4sf)__A, (__v4sf)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comige_ss (__m128 __A, __m128 __B)
{
return __builtin_ia32_comige ((__v4sf)__A, (__v4sf)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comineq_ss (__m128 __A, __m128 __B)
{
return __builtin_ia32_comineq ((__v4sf)__A, (__v4sf)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomieq_ss (__m128 __A, __m128 __B)
{
return __builtin_ia32_ucomieq ((__v4sf)__A, (__v4sf)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomilt_ss (__m128 __A, __m128 __B)
{
return __builtin_ia32_ucomilt ((__v4sf)__A, (__v4sf)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomile_ss (__m128 __A, __m128 __B)
{
return __builtin_ia32_ucomile ((__v4sf)__A, (__v4sf)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomigt_ss (__m128 __A, __m128 __B)
{
return __builtin_ia32_ucomigt ((__v4sf)__A, (__v4sf)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomige_ss (__m128 __A, __m128 __B)
{
return __builtin_ia32_ucomige ((__v4sf)__A, (__v4sf)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomineq_ss (__m128 __A, __m128 __B)
{
return __builtin_ia32_ucomineq ((__v4sf)__A, (__v4sf)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtss_si32 (__m128 __A)
{
return __builtin_ia32_cvtss2si ((__v4sf) __A);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvt_ss2si (__m128 __A)
{
return _mm_cvtss_si32 (__A);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtps_pi32 (__m128 __A)
{
return (__m64) __builtin_ia32_cvtps2pi ((__v4sf) __A);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvt_ps2pi (__m128 __A)
{
return _mm_cvtps_pi32 (__A);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvttss_si32 (__m128 __A)
{
return __builtin_ia32_cvttss2si ((__v4sf) __A);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtt_ss2si (__m128 __A)
{
return _mm_cvttss_si32 (__A);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvttps_pi32 (__m128 __A)
{
return (__m64) __builtin_ia32_cvttps2pi ((__v4sf) __A);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtt_ps2pi (__m128 __A)
{
return _mm_cvttps_pi32 (__A);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtsi32_ss (__m128 __A, int __B)
{
return (__m128) __builtin_ia32_cvtsi2ss ((__v4sf) __A, __B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvt_si2ss (__m128 __A, int __B)
{
return _mm_cvtsi32_ss (__A, __B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtpi32_ps (__m128 __A, __m64 __B)
{
return (__m128) __builtin_ia32_cvtpi2ps ((__v4sf) __A, (__v2si)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvt_pi2ps (__m128 __A, __m64 __B)
{
return _mm_cvtpi32_ps (__A, __B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtpi16_ps (__m64 __A)
{
__v4hi __sign;
__v2si __hisi, __losi;
__v4sf __zero, __ra, __rb;
__sign = __builtin_ia32_pcmpgtw ((__v4hi)0LL, (__v4hi)__A);
__losi = (__v2si) __builtin_ia32_punpcklwd ((__v4hi)__A, __sign);
__hisi = (__v2si) __builtin_ia32_punpckhwd ((__v4hi)__A, __sign);
__zero = (__v4sf) _mm_setzero_ps ();
__ra = __builtin_ia32_cvtpi2ps (__zero, __losi);
__rb = __builtin_ia32_cvtpi2ps (__ra, __hisi);
return (__m128) __builtin_ia32_movlhps (__ra, __rb);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtpu16_ps (__m64 __A)
{
__v2si __hisi, __losi;
__v4sf __zero, __ra, __rb;
__losi = (__v2si) __builtin_ia32_punpcklwd ((__v4hi)__A, (__v4hi)0LL);
__hisi = (__v2si) __builtin_ia32_punpckhwd ((__v4hi)__A, (__v4hi)0LL);
__zero = (__v4sf) _mm_setzero_ps ();
__ra = __builtin_ia32_cvtpi2ps (__zero, __losi);
__rb = __builtin_ia32_cvtpi2ps (__ra, __hisi);
return (__m128) __builtin_ia32_movlhps (__ra, __rb);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtpi8_ps (__m64 __A)
{
__v8qi __sign;
__sign = __builtin_ia32_pcmpgtb ((__v8qi)0LL, (__v8qi)__A);
__A = (__m64) __builtin_ia32_punpcklbw ((__v8qi)__A, __sign);
return _mm_cvtpi16_ps(__A);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtpu8_ps(__m64 __A)
{
__A = (__m64) __builtin_ia32_punpcklbw ((__v8qi)__A, (__v8qi)0LL);
return _mm_cvtpu16_ps(__A);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtpi32x2_ps(__m64 __A, __m64 __B)
{
__v4sf __zero = (__v4sf) _mm_setzero_ps ();
__v4sf __sfa = __builtin_ia32_cvtpi2ps (__zero, (__v2si)__A);
__v4sf __sfb = __builtin_ia32_cvtpi2ps (__sfa, (__v2si)__B);
return (__m128) __builtin_ia32_movlhps (__sfa, __sfb);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtps_pi16(__m128 __A)
{
__v4sf __hisf = (__v4sf)__A;
__v4sf __losf = __builtin_ia32_movhlps (__hisf, __hisf);
__v2si __hisi = __builtin_ia32_cvtps2pi (__hisf);
__v2si __losi = __builtin_ia32_cvtps2pi (__losf);
return (__m64) __builtin_ia32_packssdw (__hisi, __losi);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtps_pi8(__m128 __A)
{
__v4hi __tmp = (__v4hi) _mm_cvtps_pi16 (__A);
return (__m64) __builtin_ia32_packsswb (__tmp, (__v4hi)0LL);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_shuffle_ps (__m128 __A, __m128 __B, int const __mask)
{
return (__m128) __builtin_ia32_shufps ((__v4sf)__A, (__v4sf)__B, __mask);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpackhi_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_unpckhps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpacklo_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_unpcklps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loadh_pi (__m128 __A, __m64 const *__P)
{
return (__m128) __builtin_ia32_loadhps ((__v4sf)__A, (const __v2sf *)__P);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_storeh_pi (__m64 *__P, __m128 __A)
{
__builtin_ia32_storehps ((__v2sf *)__P, (__v4sf)__A);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_movehl_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_movhlps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_movelh_ps (__m128 __A, __m128 __B)
{
return (__m128) __builtin_ia32_movlhps ((__v4sf)__A, (__v4sf)__B);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loadl_pi (__m128 __A, __m64 const *__P)
{
return (__m128) __builtin_ia32_loadlps ((__v4sf)__A, (const __v2sf *)__P);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_storel_pi (__m64 *__P, __m128 __A)
{
__builtin_ia32_storelps ((__v2sf *)__P, (__v4sf)__A);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_movemask_ps (__m128 __A)
{
return __builtin_ia32_movmskps ((__v4sf)__A);
}
extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_getcsr (void)
{
return __builtin_ia32_stmxcsr ();
}
extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_MM_GET_EXCEPTION_STATE (void)
{
return _mm_getcsr() & 0x003f;
}
extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_MM_GET_EXCEPTION_MASK (void)
{
return _mm_getcsr() & 0x1f80;
}
extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_MM_GET_ROUNDING_MODE (void)
{
return _mm_getcsr() & 0x6000;
}
extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_MM_GET_FLUSH_ZERO_MODE (void)
{
return _mm_getcsr() & 0x8000;
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_setcsr (unsigned int __I)
{
__builtin_ia32_ldmxcsr (__I);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_MM_SET_EXCEPTION_STATE(unsigned int __mask)
{
_mm_setcsr((_mm_getcsr() & ~0x003f) | __mask);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_MM_SET_EXCEPTION_MASK (unsigned int __mask)
{
_mm_setcsr((_mm_getcsr() & ~0x1f80) | __mask);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_MM_SET_ROUNDING_MODE (unsigned int __mode)
{
_mm_setcsr((_mm_getcsr() & ~0x6000) | __mode);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_MM_SET_FLUSH_ZERO_MODE (unsigned int __mode)
{
_mm_setcsr((_mm_getcsr() & ~0x8000) | __mode);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set_ss (float __F)
{
return __extension__ (__m128)(__v4sf){ __F, 0.0f, 0.0f, 0.0f };
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set1_ps (float __F)
{
return __extension__ (__m128)(__v4sf){ __F, __F, __F, __F };
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set_ps1 (float __F)
{
return _mm_set1_ps (__F);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_load_ss (float const *__P)
{
return _mm_set_ss (*__P);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_load1_ps (float const *__P)
{
return _mm_set1_ps (*__P);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_load_ps1 (float const *__P)
{
return _mm_load1_ps (__P);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_load_ps (float const *__P)
{
return *(__m128 *)__P;
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loadu_ps (float const *__P)
{
return *(__m128_u *)__P;
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loadr_ps (float const *__P)
{
__v4sf __tmp = *(__v4sf *)__P;
return (__m128) __builtin_ia32_shufps (__tmp, __tmp, (((0) << 6) | ((1) << 4) | ((2) << 2) | (3)));
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set_ps (const float __Z, const float __Y, const float __X, const float __W)
{
return __extension__ (__m128)(__v4sf){ __W, __X, __Y, __Z };
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_setr_ps (float __Z, float __Y, float __X, float __W)
{
return __extension__ (__m128)(__v4sf){ __Z, __Y, __X, __W };
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_store_ss (float *__P, __m128 __A)
{
*__P = ((__v4sf)__A)[0];
}
extern __inline float __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtss_f32 (__m128 __A)
{
return ((__v4sf)__A)[0];
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_store_ps (float *__P, __m128 __A)
{
*(__m128 *)__P = __A;
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_storeu_ps (float *__P, __m128 __A)
{
*(__m128_u *)__P = __A;
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_store1_ps (float *__P, __m128 __A)
{
__v4sf __va = (__v4sf)__A;
__v4sf __tmp = __builtin_ia32_shufps (__va, __va, (((0) << 6) | ((0) << 4) | ((0) << 2) | (0)));
_mm_storeu_ps (__P, __tmp);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_store_ps1 (float *__P, __m128 __A)
{
_mm_store1_ps (__P, __A);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_storer_ps (float *__P, __m128 __A)
{
__v4sf __va = (__v4sf)__A;
__v4sf __tmp = __builtin_ia32_shufps (__va, __va, (((0) << 6) | ((1) << 4) | ((2) << 2) | (3)));
_mm_store_ps (__P, __tmp);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_move_ss (__m128 __A, __m128 __B)
{
return (__m128) __builtin_shuffle ((__v4sf)__A, (__v4sf)__B,
__extension__
(__attribute__((__vector_size__ (16))) int)
{4,1,2,3});
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_extract_pi16 (__m64 const __A, int const __N)
{
return (unsigned short) __builtin_ia32_vec_ext_v4hi ((__v4hi)__A, __N);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pextrw (__m64 const __A, int const __N)
{
return _mm_extract_pi16 (__A, __N);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_insert_pi16 (__m64 const __A, int const __D, int const __N)
{
return (__m64) __builtin_ia32_vec_set_v4hi ((__v4hi)__A, __D, __N);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pinsrw (__m64 const __A, int const __D, int const __N)
{
return _mm_insert_pi16 (__A, __D, __N);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_max_pi16 (__m64 __A, __m64 __B)
{
return (__m64) __builtin_ia32_pmaxsw ((__v4hi)__A, (__v4hi)__B);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pmaxsw (__m64 __A, __m64 __B)
{
return _mm_max_pi16 (__A, __B);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_max_pu8 (__m64 __A, __m64 __B)
{
return (__m64) __builtin_ia32_pmaxub ((__v8qi)__A, (__v8qi)__B);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pmaxub (__m64 __A, __m64 __B)
{
return _mm_max_pu8 (__A, __B);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_min_pi16 (__m64 __A, __m64 __B)
{
return (__m64) __builtin_ia32_pminsw ((__v4hi)__A, (__v4hi)__B);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pminsw (__m64 __A, __m64 __B)
{
return _mm_min_pi16 (__A, __B);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_min_pu8 (__m64 __A, __m64 __B)
{
return (__m64) __builtin_ia32_pminub ((__v8qi)__A, (__v8qi)__B);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pminub (__m64 __A, __m64 __B)
{
return _mm_min_pu8 (__A, __B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_movemask_pi8 (__m64 __A)
{
return __builtin_ia32_pmovmskb ((__v8qi)__A);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pmovmskb (__m64 __A)
{
return _mm_movemask_pi8 (__A);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mulhi_pu16 (__m64 __A, __m64 __B)
{
return (__m64) __builtin_ia32_pmulhuw ((__v4hi)__A, (__v4hi)__B);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pmulhuw (__m64 __A, __m64 __B)
{
return _mm_mulhi_pu16 (__A, __B);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_shuffle_pi16 (__m64 __A, int const __N)
{
return (__m64) __builtin_ia32_pshufw ((__v4hi)__A, __N);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pshufw (__m64 __A, int const __N)
{
return _mm_shuffle_pi16 (__A, __N);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_maskmove_si64 (__m64 __A, __m64 __N, char *__P)
{
__builtin_ia32_maskmovq ((__v8qi)__A, (__v8qi)__N, __P);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_maskmovq (__m64 __A, __m64 __N, char *__P)
{
_mm_maskmove_si64 (__A, __N, __P);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_avg_pu8 (__m64 __A, __m64 __B)
{
return (__m64) __builtin_ia32_pavgb ((__v8qi)__A, (__v8qi)__B);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pavgb (__m64 __A, __m64 __B)
{
return _mm_avg_pu8 (__A, __B);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_avg_pu16 (__m64 __A, __m64 __B)
{
return (__m64) __builtin_ia32_pavgw ((__v4hi)__A, (__v4hi)__B);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_pavgw (__m64 __A, __m64 __B)
{
return _mm_avg_pu16 (__A, __B);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sad_pu8 (__m64 __A, __m64 __B)
{
return (__m64) __builtin_ia32_psadbw ((__v8qi)__A, (__v8qi)__B);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_m_psadbw (__m64 __A, __m64 __B)
{
return _mm_sad_pu8 (__A, __B);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_stream_pi (__m64 *__P, __m64 __A)
{
__builtin_ia32_movntq ((unsigned long long *)__P, (unsigned long long)__A);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_stream_ps (float *__P, __m128 __A)
{
__builtin_ia32_movntps (__P, (__v4sf)__A);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sfence (void)
{
__builtin_ia32_sfence ();
}
#pragma GCC pop_options
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_pause (void)
{
__builtin_ia32_pause ();
}
#pragma GCC push_options
#pragma GCC target("sse2")
typedef double __v2df __attribute__ ((__vector_size__ (16)));
typedef long long __v2di __attribute__ ((__vector_size__ (16)));
typedef unsigned long long __v2du __attribute__ ((__vector_size__ (16)));
typedef int __v4si __attribute__ ((__vector_size__ (16)));
typedef unsigned int __v4su __attribute__ ((__vector_size__ (16)));
typedef short __v8hi __attribute__ ((__vector_size__ (16)));
typedef unsigned short __v8hu __attribute__ ((__vector_size__ (16)));
typedef char __v16qi __attribute__ ((__vector_size__ (16)));
typedef signed char __v16qs __attribute__ ((__vector_size__ (16)));
typedef unsigned char __v16qu __attribute__ ((__vector_size__ (16)));
typedef long long __m128i __attribute__ ((__vector_size__ (16), __may_alias__));
typedef double __m128d __attribute__ ((__vector_size__ (16), __may_alias__));
typedef long long __m128i_u __attribute__ ((__vector_size__ (16), __may_alias__, __aligned__ (1)));
typedef double __m128d_u __attribute__ ((__vector_size__ (16), __may_alias__, __aligned__ (1)));
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set_sd (double __F)
{
return __extension__ (__m128d){ __F, 0.0 };
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set1_pd (double __F)
{
return __extension__ (__m128d){ __F, __F };
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set_pd1 (double __F)
{
return _mm_set1_pd (__F);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set_pd (double __W, double __X)
{
return __extension__ (__m128d){ __X, __W };
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_setr_pd (double __W, double __X)
{
return __extension__ (__m128d){ __W, __X };
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_undefined_pd (void)
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Winit-self"
__m128d __Y = __Y;
#pragma GCC diagnostic pop
return __Y;
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_setzero_pd (void)
{
return __extension__ (__m128d){ 0.0, 0.0 };
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_move_sd (__m128d __A, __m128d __B)
{
return __extension__ (__m128d) __builtin_shuffle ((__v2df)__A, (__v2df)__B, (__v2di){2, 1});
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_load_pd (double const *__P)
{
return *(__m128d *)__P;
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loadu_pd (double const *__P)
{
return *(__m128d_u *)__P;
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_load1_pd (double const *__P)
{
return _mm_set1_pd (*__P);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_load_sd (double const *__P)
{
return _mm_set_sd (*__P);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_load_pd1 (double const *__P)
{
return _mm_load1_pd (__P);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loadr_pd (double const *__P)
{
__m128d __tmp = _mm_load_pd (__P);
return __builtin_ia32_shufpd (__tmp, __tmp, (((0) << 1) | (1)));
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_store_pd (double *__P, __m128d __A)
{
*(__m128d *)__P = __A;
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_storeu_pd (double *__P, __m128d __A)
{
*(__m128d_u *)__P = __A;
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_store_sd (double *__P, __m128d __A)
{
*__P = ((__v2df)__A)[0];
}
extern __inline double __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtsd_f64 (__m128d __A)
{
return ((__v2df)__A)[0];
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_storel_pd (double *__P, __m128d __A)
{
_mm_store_sd (__P, __A);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_storeh_pd (double *__P, __m128d __A)
{
*__P = ((__v2df)__A)[1];
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_store1_pd (double *__P, __m128d __A)
{
_mm_store_pd (__P, __builtin_ia32_shufpd (__A, __A, (((0) << 1) | (0))));
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_store_pd1 (double *__P, __m128d __A)
{
_mm_store1_pd (__P, __A);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_storer_pd (double *__P, __m128d __A)
{
_mm_store_pd (__P, __builtin_ia32_shufpd (__A, __A, (((0) << 1) | (1))));
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtsi128_si32 (__m128i __A)
{
return __builtin_ia32_vec_ext_v4si ((__v4si)__A, 0);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_add_pd (__m128d __A, __m128d __B)
{
return (__m128d) ((__v2df)__A + (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_add_sd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_addsd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sub_pd (__m128d __A, __m128d __B)
{
return (__m128d) ((__v2df)__A - (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sub_sd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_subsd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mul_pd (__m128d __A, __m128d __B)
{
return (__m128d) ((__v2df)__A * (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mul_sd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_mulsd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_div_pd (__m128d __A, __m128d __B)
{
return (__m128d) ((__v2df)__A / (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_div_sd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_divsd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sqrt_pd (__m128d __A)
{
return (__m128d)__builtin_ia32_sqrtpd ((__v2df)__A);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sqrt_sd (__m128d __A, __m128d __B)
{
__v2df __tmp = __builtin_ia32_movsd ((__v2df)__A, (__v2df)__B);
return (__m128d)__builtin_ia32_sqrtsd ((__v2df)__tmp);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_min_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_minpd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_min_sd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_minsd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_max_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_maxpd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_max_sd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_maxsd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_and_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_andpd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_andnot_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_andnpd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_or_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_orpd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_xor_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_xorpd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpeq_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpeqpd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmplt_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpltpd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmple_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmplepd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpgt_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpgtpd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpge_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpgepd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpneq_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpneqpd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnlt_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpnltpd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnle_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpnlepd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpngt_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpngtpd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnge_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpngepd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpord_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpordpd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpunord_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpunordpd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpeq_sd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpeqsd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmplt_sd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpltsd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmple_sd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmplesd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpgt_sd (__m128d __A, __m128d __B)
{
return (__m128d) __builtin_ia32_movsd ((__v2df) __A,
(__v2df)
__builtin_ia32_cmpltsd ((__v2df) __B,
(__v2df)
__A));
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpge_sd (__m128d __A, __m128d __B)
{
return (__m128d) __builtin_ia32_movsd ((__v2df) __A,
(__v2df)
__builtin_ia32_cmplesd ((__v2df) __B,
(__v2df)
__A));
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpneq_sd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpneqsd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnlt_sd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpnltsd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnle_sd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpnlesd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpngt_sd (__m128d __A, __m128d __B)
{
return (__m128d) __builtin_ia32_movsd ((__v2df) __A,
(__v2df)
__builtin_ia32_cmpnltsd ((__v2df) __B,
(__v2df)
__A));
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpnge_sd (__m128d __A, __m128d __B)
{
return (__m128d) __builtin_ia32_movsd ((__v2df) __A,
(__v2df)
__builtin_ia32_cmpnlesd ((__v2df) __B,
(__v2df)
__A));
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpord_sd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpordsd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpunord_sd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_cmpunordsd ((__v2df)__A, (__v2df)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comieq_sd (__m128d __A, __m128d __B)
{
return __builtin_ia32_comisdeq ((__v2df)__A, (__v2df)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comilt_sd (__m128d __A, __m128d __B)
{
return __builtin_ia32_comisdlt ((__v2df)__A, (__v2df)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comile_sd (__m128d __A, __m128d __B)
{
return __builtin_ia32_comisdle ((__v2df)__A, (__v2df)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comigt_sd (__m128d __A, __m128d __B)
{
return __builtin_ia32_comisdgt ((__v2df)__A, (__v2df)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comige_sd (__m128d __A, __m128d __B)
{
return __builtin_ia32_comisdge ((__v2df)__A, (__v2df)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_comineq_sd (__m128d __A, __m128d __B)
{
return __builtin_ia32_comisdneq ((__v2df)__A, (__v2df)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomieq_sd (__m128d __A, __m128d __B)
{
return __builtin_ia32_ucomisdeq ((__v2df)__A, (__v2df)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomilt_sd (__m128d __A, __m128d __B)
{
return __builtin_ia32_ucomisdlt ((__v2df)__A, (__v2df)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomile_sd (__m128d __A, __m128d __B)
{
return __builtin_ia32_ucomisdle ((__v2df)__A, (__v2df)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomigt_sd (__m128d __A, __m128d __B)
{
return __builtin_ia32_ucomisdgt ((__v2df)__A, (__v2df)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomige_sd (__m128d __A, __m128d __B)
{
return __builtin_ia32_ucomisdge ((__v2df)__A, (__v2df)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_ucomineq_sd (__m128d __A, __m128d __B)
{
return __builtin_ia32_ucomisdneq ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set_epi64x (long long __q1, long long __q0)
{
return __extension__ (__m128i)(__v2di){ __q0, __q1 };
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set_epi64 (__m64 __q1, __m64 __q0)
{
return _mm_set_epi64x ((long long)__q1, (long long)__q0);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set_epi32 (int __q3, int __q2, int __q1, int __q0)
{
return __extension__ (__m128i)(__v4si){ __q0, __q1, __q2, __q3 };
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set_epi16 (short __q7, short __q6, short __q5, short __q4,
short __q3, short __q2, short __q1, short __q0)
{
return __extension__ (__m128i)(__v8hi){
__q0, __q1, __q2, __q3, __q4, __q5, __q6, __q7 };
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set_epi8 (char __q15, char __q14, char __q13, char __q12,
char __q11, char __q10, char __q09, char __q08,
char __q07, char __q06, char __q05, char __q04,
char __q03, char __q02, char __q01, char __q00)
{
return __extension__ (__m128i)(__v16qi){
__q00, __q01, __q02, __q03, __q04, __q05, __q06, __q07,
__q08, __q09, __q10, __q11, __q12, __q13, __q14, __q15
};
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set1_epi64x (long long __A)
{
return _mm_set_epi64x (__A, __A);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set1_epi64 (__m64 __A)
{
return _mm_set_epi64 (__A, __A);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set1_epi32 (int __A)
{
return _mm_set_epi32 (__A, __A, __A, __A);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set1_epi16 (short __A)
{
return _mm_set_epi16 (__A, __A, __A, __A, __A, __A, __A, __A);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_set1_epi8 (char __A)
{
return _mm_set_epi8 (__A, __A, __A, __A, __A, __A, __A, __A,
__A, __A, __A, __A, __A, __A, __A, __A);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_setr_epi64 (__m64 __q0, __m64 __q1)
{
return _mm_set_epi64 (__q1, __q0);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_setr_epi32 (int __q0, int __q1, int __q2, int __q3)
{
return _mm_set_epi32 (__q3, __q2, __q1, __q0);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_setr_epi16 (short __q0, short __q1, short __q2, short __q3,
short __q4, short __q5, short __q6, short __q7)
{
return _mm_set_epi16 (__q7, __q6, __q5, __q4, __q3, __q2, __q1, __q0);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_setr_epi8 (char __q00, char __q01, char __q02, char __q03,
char __q04, char __q05, char __q06, char __q07,
char __q08, char __q09, char __q10, char __q11,
char __q12, char __q13, char __q14, char __q15)
{
return _mm_set_epi8 (__q15, __q14, __q13, __q12, __q11, __q10, __q09, __q08,
__q07, __q06, __q05, __q04, __q03, __q02, __q01, __q00);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_load_si128 (__m128i const *__P)
{
return *__P;
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loadu_si128 (__m128i_u const *__P)
{
return *__P;
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loadl_epi64 (__m128i_u const *__P)
{
return _mm_set_epi64 ((__m64)0LL, *(__m64_u *)__P);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loadu_si64 (void const *__P)
{
return _mm_loadl_epi64 ((__m128i_u *)__P);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loadu_si32 (void const *__P)
{
return _mm_set_epi32 (0, 0, 0, (*(__m32_u *)__P)[0]);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loadu_si16 (void const *__P)
{
return _mm_set_epi16 (0, 0, 0, 0, 0, 0, 0, (*(__m16_u *)__P)[0]);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_store_si128 (__m128i *__P, __m128i __B)
{
*__P = __B;
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_storeu_si128 (__m128i_u *__P, __m128i __B)
{
*__P = __B;
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_storel_epi64 (__m128i_u *__P, __m128i __B)
{
*(__m64_u *)__P = (__m64) ((__v2di)__B)[0];
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_storeu_si64 (void *__P, __m128i __B)
{
_mm_storel_epi64 ((__m128i_u *)__P, __B);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_storeu_si32 (void *__P, __m128i __B)
{
*(__m32_u *)__P = (__m32) ((__v4si)__B)[0];
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_storeu_si16 (void *__P, __m128i __B)
{
*(__m16_u *)__P = (__m16) ((__v8hi)__B)[0];
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_movepi64_pi64 (__m128i __B)
{
return (__m64) ((__v2di)__B)[0];
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_movpi64_epi64 (__m64 __A)
{
return _mm_set_epi64 ((__m64)0LL, __A);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_move_epi64 (__m128i __A)
{
return (__m128i)__builtin_ia32_movq128 ((__v2di) __A);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_undefined_si128 (void)
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Winit-self"
__m128i __Y = __Y;
#pragma GCC diagnostic pop
return __Y;
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_setzero_si128 (void)
{
return __extension__ (__m128i)(__v4si){ 0, 0, 0, 0 };
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtepi32_pd (__m128i __A)
{
return (__m128d)__builtin_ia32_cvtdq2pd ((__v4si) __A);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtepi32_ps (__m128i __A)
{
return (__m128)__builtin_ia32_cvtdq2ps ((__v4si) __A);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtpd_epi32 (__m128d __A)
{
return (__m128i)__builtin_ia32_cvtpd2dq ((__v2df) __A);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtpd_pi32 (__m128d __A)
{
return (__m64)__builtin_ia32_cvtpd2pi ((__v2df) __A);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtpd_ps (__m128d __A)
{
return (__m128)__builtin_ia32_cvtpd2ps ((__v2df) __A);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvttpd_epi32 (__m128d __A)
{
return (__m128i)__builtin_ia32_cvttpd2dq ((__v2df) __A);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvttpd_pi32 (__m128d __A)
{
return (__m64)__builtin_ia32_cvttpd2pi ((__v2df) __A);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtpi32_pd (__m64 __A)
{
return (__m128d)__builtin_ia32_cvtpi2pd ((__v2si) __A);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtps_epi32 (__m128 __A)
{
return (__m128i)__builtin_ia32_cvtps2dq ((__v4sf) __A);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvttps_epi32 (__m128 __A)
{
return (__m128i)__builtin_ia32_cvttps2dq ((__v4sf) __A);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtps_pd (__m128 __A)
{
return (__m128d)__builtin_ia32_cvtps2pd ((__v4sf) __A);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtsd_si32 (__m128d __A)
{
return __builtin_ia32_cvtsd2si ((__v2df) __A);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvttsd_si32 (__m128d __A)
{
return __builtin_ia32_cvttsd2si ((__v2df) __A);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtsd_ss (__m128 __A, __m128d __B)
{
return (__m128)__builtin_ia32_cvtsd2ss ((__v4sf) __A, (__v2df) __B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtsi32_sd (__m128d __A, int __B)
{
return (__m128d)__builtin_ia32_cvtsi2sd ((__v2df) __A, __B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtss_sd (__m128d __A, __m128 __B)
{
return (__m128d)__builtin_ia32_cvtss2sd ((__v2df) __A, (__v4sf)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_shuffle_pd(__m128d __A, __m128d __B, const int __mask)
{
return (__m128d)__builtin_ia32_shufpd ((__v2df)__A, (__v2df)__B, __mask);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpackhi_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_unpckhpd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpacklo_pd (__m128d __A, __m128d __B)
{
return (__m128d)__builtin_ia32_unpcklpd ((__v2df)__A, (__v2df)__B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loadh_pd (__m128d __A, double const *__B)
{
return (__m128d)__builtin_ia32_loadhpd ((__v2df)__A, __B);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loadl_pd (__m128d __A, double const *__B)
{
return (__m128d)__builtin_ia32_loadlpd ((__v2df)__A, __B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_movemask_pd (__m128d __A)
{
return __builtin_ia32_movmskpd ((__v2df)__A);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_packs_epi16 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_packsswb128 ((__v8hi)__A, (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_packs_epi32 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_packssdw128 ((__v4si)__A, (__v4si)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_packus_epi16 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_packuswb128 ((__v8hi)__A, (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpackhi_epi8 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_punpckhbw128 ((__v16qi)__A, (__v16qi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpackhi_epi16 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_punpckhwd128 ((__v8hi)__A, (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpackhi_epi32 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_punpckhdq128 ((__v4si)__A, (__v4si)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpackhi_epi64 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_punpckhqdq128 ((__v2di)__A, (__v2di)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpacklo_epi8 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_punpcklbw128 ((__v16qi)__A, (__v16qi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpacklo_epi16 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_punpcklwd128 ((__v8hi)__A, (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpacklo_epi32 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_punpckldq128 ((__v4si)__A, (__v4si)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_unpacklo_epi64 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_punpcklqdq128 ((__v2di)__A, (__v2di)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_add_epi8 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v16qu)__A + (__v16qu)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_add_epi16 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v8hu)__A + (__v8hu)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_add_epi32 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v4su)__A + (__v4su)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_add_epi64 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v2du)__A + (__v2du)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_adds_epi8 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_paddsb128 ((__v16qi)__A, (__v16qi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_adds_epi16 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_paddsw128 ((__v8hi)__A, (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_adds_epu8 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_paddusb128 ((__v16qi)__A, (__v16qi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_adds_epu16 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_paddusw128 ((__v8hi)__A, (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sub_epi8 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v16qu)__A - (__v16qu)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sub_epi16 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v8hu)__A - (__v8hu)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sub_epi32 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v4su)__A - (__v4su)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sub_epi64 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v2du)__A - (__v2du)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_subs_epi8 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_psubsb128 ((__v16qi)__A, (__v16qi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_subs_epi16 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_psubsw128 ((__v8hi)__A, (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_subs_epu8 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_psubusb128 ((__v16qi)__A, (__v16qi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_subs_epu16 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_psubusw128 ((__v8hi)__A, (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_madd_epi16 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_pmaddwd128 ((__v8hi)__A, (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mulhi_epi16 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_pmulhw128 ((__v8hi)__A, (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mullo_epi16 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v8hu)__A * (__v8hu)__B);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mul_su32 (__m64 __A, __m64 __B)
{
return (__m64)__builtin_ia32_pmuludq ((__v2si)__A, (__v2si)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mul_epu32 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_pmuludq128 ((__v4si)__A, (__v4si)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_slli_epi16 (__m128i __A, int __B)
{
return (__m128i)__builtin_ia32_psllwi128 ((__v8hi)__A, __B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_slli_epi32 (__m128i __A, int __B)
{
return (__m128i)__builtin_ia32_pslldi128 ((__v4si)__A, __B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_slli_epi64 (__m128i __A, int __B)
{
return (__m128i)__builtin_ia32_psllqi128 ((__v2di)__A, __B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_srai_epi16 (__m128i __A, int __B)
{
return (__m128i)__builtin_ia32_psrawi128 ((__v8hi)__A, __B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_srai_epi32 (__m128i __A, int __B)
{
return (__m128i)__builtin_ia32_psradi128 ((__v4si)__A, __B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_bsrli_si128 (__m128i __A, const int __N)
{
return (__m128i)__builtin_ia32_psrldqi128 (__A, __N * 8);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_bslli_si128 (__m128i __A, const int __N)
{
return (__m128i)__builtin_ia32_pslldqi128 (__A, __N * 8);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_srli_si128 (__m128i __A, const int __N)
{
return (__m128i)__builtin_ia32_psrldqi128 (__A, __N * 8);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_slli_si128 (__m128i __A, const int __N)
{
return (__m128i)__builtin_ia32_pslldqi128 (__A, __N * 8);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_srli_epi16 (__m128i __A, int __B)
{
return (__m128i)__builtin_ia32_psrlwi128 ((__v8hi)__A, __B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_srli_epi32 (__m128i __A, int __B)
{
return (__m128i)__builtin_ia32_psrldi128 ((__v4si)__A, __B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_srli_epi64 (__m128i __A, int __B)
{
return (__m128i)__builtin_ia32_psrlqi128 ((__v2di)__A, __B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sll_epi16 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_psllw128((__v8hi)__A, (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sll_epi32 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_pslld128((__v4si)__A, (__v4si)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sll_epi64 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_psllq128((__v2di)__A, (__v2di)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sra_epi16 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_psraw128 ((__v8hi)__A, (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sra_epi32 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_psrad128 ((__v4si)__A, (__v4si)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_srl_epi16 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_psrlw128 ((__v8hi)__A, (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_srl_epi32 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_psrld128 ((__v4si)__A, (__v4si)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_srl_epi64 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_psrlq128 ((__v2di)__A, (__v2di)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_and_si128 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v2du)__A & (__v2du)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_andnot_si128 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_pandn128 ((__v2di)__A, (__v2di)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_or_si128 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v2du)__A | (__v2du)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_xor_si128 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v2du)__A ^ (__v2du)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpeq_epi8 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v16qi)__A == (__v16qi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpeq_epi16 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v8hi)__A == (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpeq_epi32 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v4si)__A == (__v4si)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmplt_epi8 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v16qs)__A < (__v16qs)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmplt_epi16 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v8hi)__A < (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmplt_epi32 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v4si)__A < (__v4si)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpgt_epi8 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v16qs)__A > (__v16qs)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpgt_epi16 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v8hi)__A > (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpgt_epi32 (__m128i __A, __m128i __B)
{
return (__m128i) ((__v4si)__A > (__v4si)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_extract_epi16 (__m128i const __A, int const __N)
{
return (unsigned short) __builtin_ia32_vec_ext_v8hi ((__v8hi)__A, __N);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_insert_epi16 (__m128i const __A, int const __D, int const __N)
{
return (__m128i) __builtin_ia32_vec_set_v8hi ((__v8hi)__A, __D, __N);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_max_epi16 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_pmaxsw128 ((__v8hi)__A, (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_max_epu8 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_pmaxub128 ((__v16qi)__A, (__v16qi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_min_epi16 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_pminsw128 ((__v8hi)__A, (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_min_epu8 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_pminub128 ((__v16qi)__A, (__v16qi)__B);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_movemask_epi8 (__m128i __A)
{
return __builtin_ia32_pmovmskb128 ((__v16qi)__A);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mulhi_epu16 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_pmulhuw128 ((__v8hi)__A, (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_shufflehi_epi16 (__m128i __A, const int __mask)
{
return (__m128i)__builtin_ia32_pshufhw ((__v8hi)__A, __mask);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_shufflelo_epi16 (__m128i __A, const int __mask)
{
return (__m128i)__builtin_ia32_pshuflw ((__v8hi)__A, __mask);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_shuffle_epi32 (__m128i __A, const int __mask)
{
return (__m128i)__builtin_ia32_pshufd ((__v4si)__A, __mask);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_maskmoveu_si128 (__m128i __A, __m128i __B, char *__C)
{
__builtin_ia32_maskmovdqu ((__v16qi)__A, (__v16qi)__B, __C);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_avg_epu8 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_pavgb128 ((__v16qi)__A, (__v16qi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_avg_epu16 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_pavgw128 ((__v8hi)__A, (__v8hi)__B);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sad_epu8 (__m128i __A, __m128i __B)
{
return (__m128i)__builtin_ia32_psadbw128 ((__v16qi)__A, (__v16qi)__B);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_stream_si32 (int *__A, int __B)
{
__builtin_ia32_movnti (__A, __B);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_stream_si128 (__m128i *__A, __m128i __B)
{
__builtin_ia32_movntdq ((__v2di *)__A, (__v2di)__B);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_stream_pd (double *__A, __m128d __B)
{
__builtin_ia32_movntpd (__A, (__v2df)__B);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_clflush (void const *__A)
{
__builtin_ia32_clflush (__A);
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_lfence (void)
{
__builtin_ia32_lfence ();
}
extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mfence (void)
{
__builtin_ia32_mfence ();
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtsi32_si128 (int __A)
{
return _mm_set_epi32 (0, 0, 0, __A);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_castpd_ps(__m128d __A)
{
return (__m128) __A;
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_castpd_si128(__m128d __A)
{
return (__m128i) __A;
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_castps_pd(__m128 __A)
{
return (__m128d) __A;
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_castps_si128(__m128 __A)
{
return (__m128i) __A;
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_castsi128_ps(__m128i __A)
{
return (__m128) __A;
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_castsi128_pd(__m128i __A)
{
return (__m128d) __A;
}
#pragma GCC pop_options
extern "C" {
extern void *memcpy (void *__restrict __dest, const void *__restrict __src,
size_t __n) noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern void *memmove (void *__dest, const void *__src, size_t __n)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern void *memccpy (void *__restrict __dest, const void *__restrict __src,
int __c, size_t __n)
noexcept (true) __attribute__ ((__nonnull__ (1, 2))) __attribute__ ((__access__ (__write_only__, 1, 4)));
extern void *memset (void *__s, int __c, size_t __n) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int memcmp (const void *__s1, const void *__s2, size_t __n)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern int __memcmpeq (const void *__s1, const void *__s2, size_t __n)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern "C++"
{
extern void *memchr (void *__s, int __c, size_t __n)
noexcept (true) __asm ("memchr") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)));
extern const void *memchr (const void *__s, int __c, size_t __n)
noexcept (true) __asm ("memchr") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)));
extern __inline __attribute__ ((__always_inline__)) __attribute__ ((__gnu_inline__)) void *
memchr (void *__s, int __c, size_t __n) noexcept (true)
{
return __builtin_memchr (__s, __c, __n);
}
extern __inline __attribute__ ((__always_inline__)) __attribute__ ((__gnu_inline__)) const void *
memchr (const void *__s, int __c, size_t __n) noexcept (true)
{
return __builtin_memchr (__s, __c, __n);
}
}
extern "C++" void *rawmemchr (void *__s, int __c)
noexcept (true) __asm ("rawmemchr") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)));
extern "C++" const void *rawmemchr (const void *__s, int __c)
noexcept (true) __asm ("rawmemchr") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)));
extern "C++" void *memrchr (void *__s, int __c, size_t __n)
noexcept (true) __asm ("memrchr") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)))
__attribute__ ((__access__ (__read_only__, 1, 3)));
extern "C++" const void *memrchr (const void *__s, int __c, size_t __n)
noexcept (true) __asm ("memrchr") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)))
__attribute__ ((__access__ (__read_only__, 1, 3)));
extern char *strcpy (char *__restrict __dest, const char *__restrict __src)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern char *strncpy (char *__restrict __dest,
const char *__restrict __src, size_t __n)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern char *strcat (char *__restrict __dest, const char *__restrict __src)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern char *strncat (char *__restrict __dest, const char *__restrict __src,
size_t __n) noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int strcmp (const char *__s1, const char *__s2)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern int strncmp (const char *__s1, const char *__s2, size_t __n)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern int strcoll (const char *__s1, const char *__s2)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern size_t strxfrm (char *__restrict __dest,
const char *__restrict __src, size_t __n)
noexcept (true) __attribute__ ((__nonnull__ (2))) __attribute__ ((__access__ (__write_only__, 1, 3)));
extern int strcoll_l (const char *__s1, const char *__s2, locale_t __l)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2, 3)));
extern size_t strxfrm_l (char *__dest, const char *__src, size_t __n,
locale_t __l) noexcept (true) __attribute__ ((__nonnull__ (2, 4)))
__attribute__ ((__access__ (__write_only__, 1, 3)));
extern char *strdup (const char *__s)
noexcept (true) __attribute__ ((__malloc__)) __attribute__ ((__nonnull__ (1)));
extern char *strndup (const char *__string, size_t __n)
noexcept (true) __attribute__ ((__malloc__)) __attribute__ ((__nonnull__ (1)));
extern "C++"
{
extern char *strchr (char *__s, int __c)
noexcept (true) __asm ("strchr") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)));
extern const char *strchr (const char *__s, int __c)
noexcept (true) __asm ("strchr") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)));
extern __inline __attribute__ ((__always_inline__)) __attribute__ ((__gnu_inline__)) char *
strchr (char *__s, int __c) noexcept (true)
{
return __builtin_strchr (__s, __c);
}
extern __inline __attribute__ ((__always_inline__)) __attribute__ ((__gnu_inline__)) const char *
strchr (const char *__s, int __c) noexcept (true)
{
return __builtin_strchr (__s, __c);
}
}
extern "C++"
{
extern char *strrchr (char *__s, int __c)
noexcept (true) __asm ("strrchr") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)));
extern const char *strrchr (const char *__s, int __c)
noexcept (true) __asm ("strrchr") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)));
extern __inline __attribute__ ((__always_inline__)) __attribute__ ((__gnu_inline__)) char *
strrchr (char *__s, int __c) noexcept (true)
{
return __builtin_strrchr (__s, __c);
}
extern __inline __attribute__ ((__always_inline__)) __attribute__ ((__gnu_inline__)) const char *
strrchr (const char *__s, int __c) noexcept (true)
{
return __builtin_strrchr (__s, __c);
}
}
extern "C++" char *strchrnul (char *__s, int __c)
noexcept (true) __asm ("strchrnul") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)));
extern "C++" const char *strchrnul (const char *__s, int __c)
noexcept (true) __asm ("strchrnul") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)));
extern size_t strcspn (const char *__s, const char *__reject)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern size_t strspn (const char *__s, const char *__accept)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern "C++"
{
extern char *strpbrk (char *__s, const char *__accept)
noexcept (true) __asm ("strpbrk") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern const char *strpbrk (const char *__s, const char *__accept)
noexcept (true) __asm ("strpbrk") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern __inline __attribute__ ((__always_inline__)) __attribute__ ((__gnu_inline__)) char *
strpbrk (char *__s, const char *__accept) noexcept (true)
{
return __builtin_strpbrk (__s, __accept);
}
extern __inline __attribute__ ((__always_inline__)) __attribute__ ((__gnu_inline__)) const char *
strpbrk (const char *__s, const char *__accept) noexcept (true)
{
return __builtin_strpbrk (__s, __accept);
}
}
extern "C++"
{
extern char *strstr (char *__haystack, const char *__needle)
noexcept (true) __asm ("strstr") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern const char *strstr (const char *__haystack, const char *__needle)
noexcept (true) __asm ("strstr") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern __inline __attribute__ ((__always_inline__)) __attribute__ ((__gnu_inline__)) char *
strstr (char *__haystack, const char *__needle) noexcept (true)
{
return __builtin_strstr (__haystack, __needle);
}
extern __inline __attribute__ ((__always_inline__)) __attribute__ ((__gnu_inline__)) const char *
strstr (const char *__haystack, const char *__needle) noexcept (true)
{
return __builtin_strstr (__haystack, __needle);
}
}
extern char *strtok (char *__restrict __s, const char *__restrict __delim)
noexcept (true) __attribute__ ((__nonnull__ (2)));
extern char *__strtok_r (char *__restrict __s,
const char *__restrict __delim,
char **__restrict __save_ptr)
noexcept (true) __attribute__ ((__nonnull__ (2, 3)));
extern char *strtok_r (char *__restrict __s, const char *__restrict __delim,
char **__restrict __save_ptr)
noexcept (true) __attribute__ ((__nonnull__ (2, 3)));
extern "C++" char *strcasestr (char *__haystack, const char *__needle)
noexcept (true) __asm ("strcasestr") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern "C++" const char *strcasestr (const char *__haystack,
const char *__needle)
noexcept (true) __asm ("strcasestr") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern void *memmem (const void *__haystack, size_t __haystacklen,
const void *__needle, size_t __needlelen)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 3)))
__attribute__ ((__access__ (__read_only__, 1, 2)))
__attribute__ ((__access__ (__read_only__, 3, 4)));
extern void *__mempcpy (void *__restrict __dest,
const void *__restrict __src, size_t __n)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern void *mempcpy (void *__restrict __dest,
const void *__restrict __src, size_t __n)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern size_t strlen (const char *__s)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)));
extern size_t strnlen (const char *__string, size_t __maxlen)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)));
extern char *strerror (int __errnum) noexcept (true);
extern char *strerror_r (int __errnum, char *__buf, size_t __buflen)
noexcept (true) __attribute__ ((__nonnull__ (2))) __attribute__ ((__access__ (__write_only__, 2, 3)));
extern const char *strerrordesc_np (int __err) noexcept (true);
extern const char *strerrorname_np (int __err) noexcept (true);
extern char *strerror_l (int __errnum, locale_t __l) noexcept (true);
extern "C" {
extern int bcmp (const void *__s1, const void *__s2, size_t __n)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern void bcopy (const void *__src, void *__dest, size_t __n)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern void bzero (void *__s, size_t __n) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern "C++"
{
extern char *index (char *__s, int __c)
noexcept (true) __asm ("index") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)));
extern const char *index (const char *__s, int __c)
noexcept (true) __asm ("index") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)));
extern __inline __attribute__ ((__always_inline__)) __attribute__ ((__gnu_inline__)) char *
index (char *__s, int __c) noexcept (true)
{
return __builtin_index (__s, __c);
}
extern __inline __attribute__ ((__always_inline__)) __attribute__ ((__gnu_inline__)) const char *
index (const char *__s, int __c) noexcept (true)
{
return __builtin_index (__s, __c);
}
}
extern "C++"
{
extern char *rindex (char *__s, int __c)
noexcept (true) __asm ("rindex") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)));
extern const char *rindex (const char *__s, int __c)
noexcept (true) __asm ("rindex") __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)));
extern __inline __attribute__ ((__always_inline__)) __attribute__ ((__gnu_inline__)) char *
rindex (char *__s, int __c) noexcept (true)
{
return __builtin_rindex (__s, __c);
}
extern __inline __attribute__ ((__always_inline__)) __attribute__ ((__gnu_inline__)) const char *
rindex (const char *__s, int __c) noexcept (true)
{
return __builtin_rindex (__s, __c);
}
}
extern int ffs (int __i) noexcept (true) __attribute__ ((__const__));
extern int ffsl (long int __l) noexcept (true) __attribute__ ((__const__));
__extension__ extern int ffsll (long long int __ll)
noexcept (true) __attribute__ ((__const__));
extern int strcasecmp (const char *__s1, const char *__s2)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern int strncasecmp (const char *__s1, const char *__s2, size_t __n)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern int strcasecmp_l (const char *__s1, const char *__s2, locale_t __loc)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2, 3)));
extern int strncasecmp_l (const char *__s1, const char *__s2,
size_t __n, locale_t __loc)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2, 4)));
}
extern void explicit_bzero (void *__s, size_t __n) noexcept (true) __attribute__ ((__nonnull__ (1)))
__attribute__ ((__access__ (__write_only__, 1, 2)));
extern char *strsep (char **__restrict __stringp,
const char *__restrict __delim)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern char *strsignal (int __sig) noexcept (true);
extern const char *sigabbrev_np (int __sig) noexcept (true);
extern const char *sigdescr_np (int __sig) noexcept (true);
extern char *__stpcpy (char *__restrict __dest, const char *__restrict __src)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern char *stpcpy (char *__restrict __dest, const char *__restrict __src)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern char *__stpncpy (char *__restrict __dest,
const char *__restrict __src, size_t __n)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern char *stpncpy (char *__restrict __dest,
const char *__restrict __src, size_t __n)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int strverscmp (const char *__s1, const char *__s2)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern char *strfry (char *__string) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern void *memfrob (void *__s, size_t __n) noexcept (true) __attribute__ ((__nonnull__ (1)))
__attribute__ ((__access__ (__read_write__, 1, 2)));
extern "C++" char *basename (char *__filename)
noexcept (true) __asm ("basename") __attribute__ ((__nonnull__ (1)));
extern "C++" const char *basename (const char *__filename)
noexcept (true) __asm ("basename") __attribute__ ((__nonnull__ (1)));
}
#pragma GCC push_options
#pragma GCC target "sse2"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSE2 {
template <class V>
using VecArg = V;
namespace detail {
constexpr size_t ScaleByPower(size_t N, int pow2) {
return pow2 >= 0 ? (N << pow2) : (N >> (-pow2));
}
template <typename T>
inline __attribute__((always_inline)) void MaybeUnpoison(T* __restrict__ unaligned, size_t count) {
(void)unaligned;
(void)count;
}
}
template <typename Lane, size_t N, int kPow2>
struct Simd {
constexpr Simd() = default;
using T = Lane;
private:
static_assert(sizeof(Lane) <= 8, "Lanes are up to 64-bit");
static constexpr size_t kWhole = N & 0xFFFFF;
static constexpr int kFrac = static_cast<int>(N >> 20);
static_assert(kWhole <= 8 * 65536 && kFrac <= 3, "Out of range");
static_assert(kFrac == 0 || kWhole == 1, "If frac, whole must be 1");
static_assert((kWhole & (kWhole - 1)) == 0 && kWhole != 0, "Not 2^x");
static_assert(kPow2 >= -8, "Forgot kPow2 recursion terminator?");
public:
static constexpr size_t kPrivateLanes =
((size_t{1}) > (detail::ScaleByPower(kWhole, kPow2 - kFrac)) ? (size_t{1}) : (detail::ScaleByPower(kWhole, kPow2 - kFrac)));
constexpr size_t MaxLanes() const { return kPrivateLanes; }
constexpr size_t MaxBytes() const { return kPrivateLanes * sizeof(Lane); }
constexpr size_t MaxBlocks() const { return (MaxBytes() + 15) / 16; }
constexpr int Pow2() const { return kPow2; }
template <typename NewT>
static constexpr size_t RepartitionLanes() {
return (kPrivateLanes * sizeof(T) + sizeof(NewT) - 1) / sizeof(NewT);
}
template <typename NewT>
static constexpr int RebindPow2() {
return kPow2 +
((sizeof(NewT) >= sizeof(T))
? static_cast<int>(CeilLog2(sizeof(NewT) / sizeof(T)))
: -static_cast<int>(CeilLog2(sizeof(T) / sizeof(NewT))));
}
private:
template <int kNewPow2, size_t kNewMaxLanes>
static constexpr size_t WholeN() {
return detail::ScaleByPower(kNewMaxLanes, -kNewPow2);
}
template <int kNewPow2, size_t kNewMaxLanes>
static constexpr size_t FracN() {
static_assert(65536 <= (size_t{1} << 20), "Change bit shift");
return static_cast<size_t>(
1 + (((0) > (kNewPow2 - static_cast<int>(CeilLog2(kNewMaxLanes))) ? (0) : (kNewPow2 - static_cast<int>(CeilLog2(kNewMaxLanes))))
<< 20));
}
public:
template <int kNewPow2, size_t kNewMaxLanes>
static constexpr size_t NewN() {
return WholeN<kNewPow2, kNewMaxLanes>() == 0
? FracN<kNewPow2, kNewMaxLanes>()
: WholeN<kNewPow2, kNewMaxLanes>();
}
template <typename NewT>
using Rebind =
Simd<NewT, NewN<RebindPow2<NewT>(), kPrivateLanes>(), RebindPow2<NewT>()>;
template <typename NewT>
using Repartition =
Simd<NewT, NewN<kPow2, RepartitionLanes<NewT>()>(), kPow2>;
using Half = Simd<T, N, kPow2 - 1>;
using Twice = Simd<T, N, kPow2 + 1>;
};
namespace detail {
template <typename T, size_t N, int kPow2>
constexpr bool IsFull(Simd<T, N, kPow2> ) {
return N == (16 / sizeof(T)) && kPow2 == 0;
}
template <typename T, size_t N, int kPow2>
struct ClampNAndPow2 {
using type = Simd<T, ((N) < (65536) ? (N) : (65536)), ((kPow2) < (3) ? (kPow2) : (3))>;
};
template <typename T, int kPow2>
struct ScalableTagChecker {
using type = typename ClampNAndPow2<T, (16 / sizeof(T)), kPow2>::type;
};
template <typename T, size_t kLimit, int kPow2>
struct CappedTagChecker {
static_assert(kLimit != 0, "Does not make sense to have zero lanes");
static constexpr size_t kLimitPow2 = size_t{1} << hwy::FloorLog2(kLimit);
static constexpr size_t N = ((kLimitPow2) < ((16 / sizeof(T))) ? (kLimitPow2) : ((16 / sizeof(T))));
using type = typename ClampNAndPow2<T, N, kPow2>::type;
};
template <typename T, size_t kNumLanes>
struct FixedTagChecker {
static_assert(kNumLanes != 0, "Does not make sense to have zero lanes");
static_assert(kNumLanes <= (16 / sizeof(T)), "Too many lanes");
using type = Simd<T, kNumLanes, 0>;
};
}
template <typename T, int kPow2 = 0>
using ScalableTag = typename detail::ScalableTagChecker<T, kPow2>::type;
template <typename T, size_t kLimit, int kPow2 = 0>
using CappedTag = typename detail::CappedTagChecker<T, kLimit, kPow2>::type;
template <typename T, size_t kLimit, int kPow2 = 0>
using CappedTagIfFixed = CappedTag<T, kLimit, kPow2>;
template <typename T, size_t kNumLanes>
using FixedTag = typename detail::FixedTagChecker<T, kNumLanes>::type;
template <typename T>
using Full16 = Simd<T, 2 / sizeof(T), 0>;
template <typename T>
using Full32 = Simd<T, 4 / sizeof(T), 0>;
template <typename T>
using Full64 = Simd<T, 8 / sizeof(T), 0>;
template <typename T>
using Full128 = Simd<T, 16 / sizeof(T), 0>;
template <class D>
using TFromD = typename D::T;
template <class D>
inline __attribute__((always_inline)) __attribute__((unused)) constexpr size_t MaxLanes(D) {
return D::kPrivateLanes;
}
template <class D>
inline __attribute__((always_inline)) __attribute__((unused)) constexpr size_t Lanes(D) {
return D::kPrivateLanes;
}
template <class T, class D>
using Rebind = typename D::template Rebind<T>;
template <class D>
using RebindToSigned = Rebind<MakeSigned<TFromD<D>>, D>;
template <class D>
using RebindToUnsigned = Rebind<MakeUnsigned<TFromD<D>>, D>;
template <class D>
using RebindToFloat = Rebind<MakeFloat<TFromD<D>>, D>;
template <class T, class D>
using Repartition = typename D::template Repartition<T>;
template <class D>
using RepartitionToWide = Repartition<MakeWide<TFromD<D>>, D>;
template <class D>
using RepartitionToNarrow = Repartition<MakeNarrow<TFromD<D>>, D>;
template <class D>
using Half = typename D::Half;
template <class D>
using Twice = typename D::Twice;
template <class D>
using BlockDFromD =
Simd<TFromD<D>, ((16 / sizeof(TFromD<D>)) < (D::kPrivateLanes) ? (16 / sizeof(TFromD<D>)) : (D::kPrivateLanes)), 0>;
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC push_options
#pragma GCC target "sse2"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSE2 {
namespace detail {
template <typename T>
struct Raw128 {
using type = __m128i;
};
template <>
struct Raw128<float> {
using type = __m128;
};
template <>
struct Raw128<double> {
using type = __m128d;
};
}
template <typename T, size_t N = 16 / sizeof(T)>
class Vec128 {
using Raw = typename detail::Raw128<T>::type;
public:
using PrivateT = T;
static constexpr size_t kPrivateN = N;
inline __attribute__((always_inline)) Vec128& operator*=(const Vec128 other) {
return *this = (*this * other);
}
inline __attribute__((always_inline)) Vec128& operator/=(const Vec128 other) {
return *this = (*this / other);
}
inline __attribute__((always_inline)) Vec128& operator+=(const Vec128 other) {
return *this = (*this + other);
}
inline __attribute__((always_inline)) Vec128& operator-=(const Vec128 other) {
return *this = (*this - other);
}
inline __attribute__((always_inline)) Vec128& operator&=(const Vec128 other) {
return *this = (*this & other);
}
inline __attribute__((always_inline)) Vec128& operator|=(const Vec128 other) {
return *this = (*this | other);
}
inline __attribute__((always_inline)) Vec128& operator^=(const Vec128 other) {
return *this = (*this ^ other);
}
Raw raw;
};
template <typename T>
using Vec64 = Vec128<T, 8 / sizeof(T)>;
template <typename T>
using Vec32 = Vec128<T, 4 / sizeof(T)>;
template <typename T>
using Vec16 = Vec128<T, 2 / sizeof(T)>;
template <typename T, size_t N = 16 / sizeof(T)>
struct Mask128 {
typename detail::Raw128<T>::type raw;
};
namespace detail {
template <typename T, size_t N>
constexpr uint64_t OnlyActive(uint64_t mask_bits) {
return ((N * sizeof(T)) == 16) ? mask_bits : mask_bits & ((1ull << N) - 1);
}
}
template <class V>
using DFromV = Simd<typename V::PrivateT, V::kPrivateN, 0>;
template <class V>
using TFromV = typename V::PrivateT;
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<!hwy::IsFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<TFromD<D>, D::kPrivateLanes> Zero(D ) {
return Vec128<TFromD<D>, D::kPrivateLanes>{_mm_setzero_si128()};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, D::kPrivateLanes> Zero(D ) {
return Vec128<float, D::kPrivateLanes>{_mm_setzero_ps()};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, D::kPrivateLanes> Zero(D ) {
return Vec128<double, D::kPrivateLanes>{_mm_setzero_pd()};
}
template <class D>
using VFromD = decltype(Zero(D()));
template <class D>
struct Vec2 {
VFromD<D> v0;
VFromD<D> v1;
};
template <class D>
struct Vec3 {
VFromD<D> v0;
VFromD<D> v1;
VFromD<D> v2;
};
template <class D>
struct Vec4 {
VFromD<D> v0;
VFromD<D> v1;
VFromD<D> v2;
VFromD<D> v3;
};
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec2<D> Create2(D , VFromD<D> v0, VFromD<D> v1) {
return Vec2<D>{v0, v1};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec3<D> Create3(D , VFromD<D> v0, VFromD<D> v1, VFromD<D> v2) {
return Vec3<D>{v0, v1, v2};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec4<D> Create4(D , VFromD<D> v0, VFromD<D> v1, VFromD<D> v2,
VFromD<D> v3) {
return Vec4<D>{v0, v1, v2, v3};
}
template <size_t kIndex, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Get2(Vec2<D> tuple) {
static_assert(kIndex < 2, "Tuple index out of bounds");
return kIndex == 0 ? tuple.v0 : tuple.v1;
}
template <size_t kIndex, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Get3(Vec3<D> tuple) {
static_assert(kIndex < 3, "Tuple index out of bounds");
return kIndex == 0 ? tuple.v0 : kIndex == 1 ? tuple.v1 : tuple.v2;
}
template <size_t kIndex, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Get4(Vec4<D> tuple) {
static_assert(kIndex < 4, "Tuple index out of bounds");
return kIndex == 0 ? tuple.v0
: kIndex == 1 ? tuple.v1
: kIndex == 2 ? tuple.v2
: tuple.v3;
}
namespace detail {
inline __attribute__((always_inline)) __m128i BitCastToInteger(__m128i v) { return v; }
inline __attribute__((always_inline)) __m128i BitCastToInteger(__m128 v) { return _mm_castps_si128(v); }
inline __attribute__((always_inline)) __m128i BitCastToInteger(__m128d v) { return _mm_castpd_si128(v); }
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<uint8_t, N * sizeof(T)> BitCastToByte(Vec128<T, N> v) {
return Vec128<uint8_t, N * sizeof(T)>{BitCastToInteger(v.raw)};
}
template <typename T>
struct BitCastFromInteger128 {
inline __attribute__((always_inline)) __m128i operator()(__m128i v) { return v; }
};
template <>
struct BitCastFromInteger128<float> {
inline __attribute__((always_inline)) __m128 operator()(__m128i v) { return _mm_castsi128_ps(v); }
};
template <>
struct BitCastFromInteger128<double> {
inline __attribute__((always_inline)) __m128d operator()(__m128i v) { return _mm_castsi128_pd(v); }
};
template <class D>
inline __attribute__((always_inline)) VFromD<D> BitCastFromByte(D ,
Vec128<uint8_t, D().MaxBytes()> v) {
return VFromD<D>{BitCastFromInteger128<TFromD<D>>()(v.raw)};
}
}
template <class D, typename FromT>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> BitCast(D d,
Vec128<FromT, Repartition<FromT, D>().MaxLanes()> v) {
return detail::BitCastFromByte(d, detail::BitCastToByte(v));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , TFromD<D> t) {
return VFromD<D>{_mm_set1_epi8(static_cast<char>(t))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr,
hwy::EnableIf<!hwy::IsSpecialFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , TFromD<D> t) {
return VFromD<D>{_mm_set1_epi16(static_cast<short>(t))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>() || IsSame<TFromD<D>, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , TFromD<D> t) {
return VFromD<D>{_mm_set1_epi32(static_cast<int>(t))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>() || IsSame<TFromD<D>, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , TFromD<D> t) {
return VFromD<D>{_mm_set1_epi64x(static_cast<long long>(t))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , float t) {
return VFromD<D>{_mm_set1_ps(t)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , double t) {
return VFromD<D>{_mm_set1_pd(t)};
}
template <class D, hwy::EnableIf<hwy::IsSpecialFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D df, TFromD<D> t) {
const RebindToUnsigned<decltype(df)> du;
static_assert(sizeof(TFromD<D>) == 2, "Expecting [b]f16");
uint16_t bits;
CopyBytes<2>(&t, &bits);
return BitCast(df, Set(du, bits));
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wuninitialized"
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<!hwy::IsFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Undefined(D ) {
return VFromD<D>{_mm_undefined_si128()};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Undefined(D ) {
return VFromD<D>{_mm_undefined_ps()};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Undefined(D ) {
return VFromD<D>{_mm_undefined_pd()};
}
#pragma GCC diagnostic pop
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T GetLane(const Vec128<T, N> v) {
return static_cast<T>(_mm_cvtsi128_si32(v.raw) & 0xFF);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T GetLane(const Vec128<T, N> v) {
return static_cast<T>(_mm_cvtsi128_si32(v.raw) & 0xFFFF);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T GetLane(const Vec128<T, N> v) {
return static_cast<T>(_mm_cvtsi128_si32(v.raw));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) float GetLane(const Vec128<float, N> v) {
return _mm_cvtss_f32(v.raw);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T GetLane(const Vec128<T, N> v) {
const DFromV<decltype(v)> d;
alignas(16) T lanes[2];
Store(v, d, lanes);
return lanes[0];
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) double GetLane(const Vec128<double, N> v) {
return _mm_cvtsd_f64(v.raw);
}
template <class D, class FromV, hwy::EnableIf<DFromV<FromV>::kPrivateLanes * sizeof(TFromV<FromV>) <= 16>* = nullptr,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ResizeBitCast(D d, FromV v) {
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, VFromD<decltype(du8)>{detail::BitCastToInteger(v.raw)});
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> And(Vec128<T, N> a, Vec128<T, N> b) {
return Vec128<T, N>{_mm_and_si128(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> And(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_and_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> And(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_and_pd(a.raw, b.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> AndNot(Vec128<T, N> not_mask, Vec128<T, N> mask) {
return Vec128<T, N>{_mm_andnot_si128(not_mask.raw, mask.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> AndNot(const Vec128<float, N> not_mask,
const Vec128<float, N> mask) {
return Vec128<float, N>{_mm_andnot_ps(not_mask.raw, mask.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> AndNot(const Vec128<double, N> not_mask,
const Vec128<double, N> mask) {
return Vec128<double, N>{_mm_andnot_pd(not_mask.raw, mask.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Or(Vec128<T, N> a, Vec128<T, N> b) {
return Vec128<T, N>{_mm_or_si128(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Or(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_or_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Or(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_or_pd(a.raw, b.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Xor(Vec128<T, N> a, Vec128<T, N> b) {
return Vec128<T, N>{_mm_xor_si128(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Xor(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_xor_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Xor(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_xor_pd(a.raw, b.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Not(const Vec128<T, N> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
using VU = VFromD<decltype(du)>;
return Xor(v, BitCast(d, VU{_mm_set1_epi32(-1)}));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Xor3(Vec128<T, N> x1, Vec128<T, N> x2, Vec128<T, N> x3) {
return Xor(x1, Xor(x2, x3));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Or3(Vec128<T, N> o1, Vec128<T, N> o2, Vec128<T, N> o3) {
return Or(o1, Or(o2, o3));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> OrAnd(Vec128<T, N> o, Vec128<T, N> a1, Vec128<T, N> a2) {
return Or(o, And(a1, a2));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfVecThenElse(Vec128<T, N> mask, Vec128<T, N> yes,
Vec128<T, N> no) {
return IfThenElse(MaskFromVec(mask), yes, no);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator&(const Vec128<T, N> a, const Vec128<T, N> b) {
return And(a, b);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator|(const Vec128<T, N> a, const Vec128<T, N> b) {
return Or(a, b);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator^(const Vec128<T, N> a, const Vec128<T, N> b) {
return Xor(a, b);
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Neg(hwy::FloatTag , const Vec128<T, N> v) {
return Xor(v, SignBit(DFromV<decltype(v)>()));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Neg(hwy::NonFloatTag , const Vec128<T, N> v) {
return Zero(DFromV<decltype(v)>()) - v;
}
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Neg(const Vec128<T, N> v) {
return detail::Neg(hwy::IsFloatTag<T>(), v);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Abs(const Vec128<float, N> v) {
const Vec128<int32_t, N> mask{_mm_set1_epi32(0x7FFFFFFF)};
return v & BitCast(DFromV<decltype(v)>(), mask);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Abs(const Vec128<double, N> v) {
const Vec128<int64_t, N> mask{_mm_set1_epi64x(0x7FFFFFFFFFFFFFFFLL)};
return v & BitCast(DFromV<decltype(v)>(), mask);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CopySign(const V magn, const V sign) {
static_assert(IsFloat<TFromV<V>>(), "Only makes sense for floating-point");
const DFromV<decltype(magn)> d;
const auto msb = SignBit(d);
return BitwiseIfThenElse(msb, sign, magn);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CopySignToAbs(const V abs, const V sign) {
const DFromV<decltype(abs)> d;
return OrAnd(abs, SignBit(d), sign);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> MaskFromVec(const Vec128<T, N> v) {
return Mask128<T, N>{v.raw};
}
template <class D>
using MFromD = decltype(MaskFromVec(VFromD<D>()));
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> VecFromMask(const Mask128<T, N> v) {
return Vec128<T, N>{v.raw};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> VecFromMask(D , MFromD<D> v) {
return VFromD<D>{v.raw};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfThenElse(Mask128<T, N> mask, Vec128<T, N> yes,
Vec128<T, N> no) {
const auto vmask = VecFromMask(DFromV<decltype(no)>(), mask);
return Or(And(vmask, yes), AndNot(vmask, no));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfThenElseZero(Mask128<T, N> mask, Vec128<T, N> yes) {
return yes & VecFromMask(DFromV<decltype(yes)>(), mask);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfThenZeroElse(Mask128<T, N> mask, Vec128<T, N> no) {
return AndNot(VecFromMask(DFromV<decltype(no)>(), mask), no);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> Not(const Mask128<T, N> m) {
const Simd<T, N, 0> d;
return MaskFromVec(Not(VecFromMask(d, m)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> And(const Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(And(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> AndNot(const Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(AndNot(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> Or(const Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(Or(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> Xor(const Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(Xor(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> ExclusiveNeither(const Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(AndNot(VecFromMask(d, a), Not(VecFromMask(d, b))));
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> ShiftLeft(const Vec128<uint16_t, N> v) {
return Vec128<uint16_t, N>{_mm_slli_epi16(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> ShiftLeft(const Vec128<uint32_t, N> v) {
return Vec128<uint32_t, N>{_mm_slli_epi32(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> ShiftLeft(const Vec128<uint64_t, N> v) {
return Vec128<uint64_t, N>{_mm_slli_epi64(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> ShiftLeft(const Vec128<int16_t, N> v) {
return Vec128<int16_t, N>{_mm_slli_epi16(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> ShiftLeft(const Vec128<int32_t, N> v) {
return Vec128<int32_t, N>{_mm_slli_epi32(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> ShiftLeft(const Vec128<int64_t, N> v) {
return Vec128<int64_t, N>{_mm_slli_epi64(v.raw, kBits)};
}
template <int kBits, typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftLeft(const Vec128<T, N> v) {
const DFromV<decltype(v)> d8;
const Vec128<T, N> shifted{ShiftLeft<kBits>(Vec128<MakeWide<T>>{v.raw}).raw};
return kBits == 1
? (v + v)
: (shifted & Set(d8, static_cast<T>((0xFF << kBits) & 0xFF)));
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> ShiftRight(const Vec128<uint16_t, N> v) {
return Vec128<uint16_t, N>{_mm_srli_epi16(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> ShiftRight(const Vec128<uint32_t, N> v) {
return Vec128<uint32_t, N>{_mm_srli_epi32(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> ShiftRight(const Vec128<uint64_t, N> v) {
return Vec128<uint64_t, N>{_mm_srli_epi64(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> ShiftRight(const Vec128<uint8_t, N> v) {
const DFromV<decltype(v)> d8;
const Vec128<uint8_t, N> shifted{
ShiftRight<kBits>(Vec128<uint16_t>{v.raw}).raw};
return shifted & Set(d8, 0xFF >> kBits);
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> ShiftRight(const Vec128<int16_t, N> v) {
return Vec128<int16_t, N>{_mm_srai_epi16(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> ShiftRight(const Vec128<int32_t, N> v) {
return Vec128<int32_t, N>{_mm_srai_epi32(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> ShiftRight(const Vec128<int8_t, N> v) {
const DFromV<decltype(v)> di;
const RebindToUnsigned<decltype(di)> du;
const auto shifted = BitCast(di, ShiftRight<kBits>(BitCast(du, v)));
const auto shifted_sign = BitCast(di, Set(du, 0x80 >> kBits));
return (shifted ^ shifted_sign) - shifted_sign;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Load(D , const T* __restrict__ aligned) {
return Vec128<T>{_mm_load_si128(reinterpret_cast<const __m128i*>(aligned))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> Load(D , const float* __restrict__ aligned) {
return Vec128<float>{_mm_load_ps(aligned)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> Load(D , const double* __restrict__ aligned) {
return Vec128<double>{_mm_load_pd(aligned)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> LoadU(D , const T* __restrict__ p) {
return Vec128<T>{_mm_loadu_si128(reinterpret_cast<const __m128i*>(p))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> LoadU(D , const float* __restrict__ p) {
return Vec128<float>{_mm_loadu_ps(p)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> LoadU(D , const double* __restrict__ p) {
return Vec128<double>{_mm_loadu_pd(p)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> Load(D , const T* __restrict__ p) {
return Vec64<T>{_mm_loadl_epi64(reinterpret_cast<const __m128i*>(p))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<float> Load(D , const float* __restrict__ p) {
const __m128 hi = _mm_setzero_ps();
return Vec64<float>{_mm_loadl_pi(hi, reinterpret_cast<const __m64*>(p))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<double> Load(D , const double* __restrict__ p) {
return Vec64<double>{_mm_load_sd(p)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<float> Load(D , const float* __restrict__ p) {
return Vec32<float>{_mm_load_ss(p)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Load(D d, const T* __restrict__ p) {
detail::MaybeUnpoison(p, Lanes(d));
int32_t bits = 0;
CopyBytes<d.MaxBytes()>(p, &bits);
return VFromD<D>{_mm_cvtsi32_si128(bits)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> LoadU(D d, const T* __restrict__ p) {
return Load(d, p);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> LoadDup128(D d, const T* __restrict__ p) {
return LoadU(d, p);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec128<T> v, D , T* __restrict__ aligned) {
_mm_store_si128(reinterpret_cast<__m128i*>(aligned), v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec128<float> v, D , float* __restrict__ aligned) {
_mm_store_ps(aligned, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec128<double> v, D ,
double* __restrict__ aligned) {
_mm_store_pd(aligned, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreU(Vec128<T> v, D , T* __restrict__ p) {
_mm_storeu_si128(reinterpret_cast<__m128i*>(p), v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreU(Vec128<float> v, D , float* __restrict__ p) {
_mm_storeu_ps(p, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreU(Vec128<double> v, D , double* __restrict__ p) {
_mm_storeu_pd(p, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec64<T> v, D , T* __restrict__ p) {
_mm_storel_epi64(reinterpret_cast<__m128i*>(p), v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec64<float> v, D , float* __restrict__ p) {
_mm_storel_pi(reinterpret_cast<__m64*>(p), v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec64<double> v, D , double* __restrict__ p) {
_mm_storel_pd(p, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(VFromD<D> v, D d, T* __restrict__ p) {
CopyBytes<d.MaxBytes()>(&v, p);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec32<float> v, D , float* __restrict__ p) {
_mm_store_ss(p, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreU(VFromD<D> v, D d, T* __restrict__ p) {
Store(v, d, p);
}
template <typename T, size_t N, typename TI, size_t NI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<TI, NI> TableLookupBytes(const Vec128<T, N> bytes,
const Vec128<TI, NI> from) {
typedef uint8_t GccU8RawVectType __attribute__((__vector_size__(16)));
return Vec128<TI, NI>{reinterpret_cast<typename detail::Raw128<TI>::type>(
__builtin_shuffle(reinterpret_cast<GccU8RawVectType>(bytes.raw),
reinterpret_cast<GccU8RawVectType>(from.raw)))};
}
template <class V, class VI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VI TableLookupBytesOr0(const V bytes, const VI from) {
const DFromV<decltype(from)> d;
const Repartition<int8_t, decltype(d)> di8;
const auto di8_from = BitCast(di8, from);
return BitCast(d, IfThenZeroElse(di8_from < Zero(di8),
TableLookupBytes(bytes, di8_from)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Shuffle2301(const Vec128<T, N> v) {
static_assert(sizeof(T) == 4, "Only for 32-bit lanes");
static_assert(N == 2 || N == 4, "Does not make sense for N=1");
return Vec128<T, N>{_mm_shuffle_epi32(v.raw, 0xB1)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Shuffle2301(const Vec128<float, N> v) {
static_assert(N == 2 || N == 4, "Does not make sense for N=1");
return Vec128<float, N>{_mm_shuffle_ps(v.raw, v.raw, 0xB1)};
}
namespace detail {
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<T> ShuffleTwo2301(const Vec32<T> a, const Vec32<T> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> d2;
const auto ba = Combine(d2, b, a);
Vec32<uint16_t> ba_shuffled{
_mm_shufflelo_epi16(ba.raw,
(((
3
) << 6) | ((
0
) << 4) | ((
3
) << 2) | (
0
))
)};
return BitCast(d, Or(ShiftLeft<8>(ba_shuffled), ShiftRight<8>(ba_shuffled)));
}
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ShuffleTwo2301(const Vec64<T> a, const Vec64<T> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> d2;
const auto ba = Combine(d2, b, a);
Vec64<uint32_t> ba_shuffled{
_mm_shuffle_epi32(ba.raw,
(((
3
) << 6) | ((
0
) << 4) | ((
3
) << 2) | (
0
))
)};
return Vec64<T>{
_mm_shufflelo_epi16(ba_shuffled.raw,
(((
2
) << 6) | ((
3
) << 4) | ((
0
) << 2) | (
1
))
)};
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ShuffleTwo2301(const Vec128<T> a, const Vec128<T> b) {
const DFromV<decltype(a)> d;
const RebindToFloat<decltype(d)> df;
constexpr int m =
(((
2
) << 6) | ((
3
) << 4) | ((
0
) << 2) | (
1
))
;
return BitCast(d, Vec128<float>{_mm_shuffle_ps(BitCast(df, a).raw,
BitCast(df, b).raw, m)});
}
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<T> ShuffleTwo1230(const Vec32<T> a, const Vec32<T> b) {
const DFromV<decltype(a)> d;
const auto zero = Zero(d);
const Rebind<int16_t, decltype(d)> di16;
const Vec32<int16_t> a_shuffled{_mm_shufflelo_epi16(
_mm_unpacklo_epi8(a.raw, zero.raw),
(((
3
) << 6) | ((
0
) << 4) | ((
3
) << 2) | (
0
))
)};
const Vec32<int16_t> b_shuffled{_mm_shufflelo_epi16(
_mm_unpacklo_epi8(b.raw, zero.raw),
(((
1
) << 6) | ((
2
) << 4) | ((
1
) << 2) | (
2
))
)};
const auto ba_shuffled = Combine(di16, b_shuffled, a_shuffled);
return Vec32<T>{_mm_packus_epi16(ba_shuffled.raw, ba_shuffled.raw)};
}
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ShuffleTwo1230(const Vec64<T> a, const Vec64<T> b) {
const DFromV<decltype(a)> d;
const Vec32<T> a_shuffled{
_mm_shufflelo_epi16(a.raw,
(((
3
) << 6) | ((
0
) << 4) | ((
3
) << 2) | (
0
))
)};
const Vec32<T> b_shuffled{
_mm_shufflelo_epi16(b.raw,
(((
1
) << 6) | ((
2
) << 4) | ((
1
) << 2) | (
2
))
)};
return Combine(d, b_shuffled, a_shuffled);
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ShuffleTwo1230(const Vec128<T> a, const Vec128<T> b) {
const DFromV<decltype(a)> d;
const RebindToFloat<decltype(d)> df;
constexpr int m =
(((
1
) << 6) | ((
2
) << 4) | ((
3
) << 2) | (
0
))
;
return BitCast(d, Vec128<float>{_mm_shuffle_ps(BitCast(df, a).raw,
BitCast(df, b).raw, m)});
}
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<T> ShuffleTwo3012(const Vec32<T> a, const Vec32<T> b) {
const DFromV<decltype(a)> d;
const auto zero = Zero(d);
const Rebind<int16_t, decltype(d)> di16;
const Vec32<int16_t> a_shuffled{_mm_shufflelo_epi16(
_mm_unpacklo_epi8(a.raw, zero.raw),
(((
1
) << 6) | ((
2
) << 4) | ((
1
) << 2) | (
2
))
)};
const Vec32<int16_t> b_shuffled{_mm_shufflelo_epi16(
_mm_unpacklo_epi8(b.raw, zero.raw),
(((
3
) << 6) | ((
0
) << 4) | ((
3
) << 2) | (
0
))
)};
const auto ba_shuffled = Combine(di16, b_shuffled, a_shuffled);
return Vec32<T>{_mm_packus_epi16(ba_shuffled.raw, ba_shuffled.raw)};
}
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ShuffleTwo3012(const Vec64<T> a, const Vec64<T> b) {
const DFromV<decltype(a)> d;
const Vec32<T> a_shuffled{
_mm_shufflelo_epi16(a.raw,
(((
1
) << 6) | ((
2
) << 4) | ((
1
) << 2) | (
2
))
)};
const Vec32<T> b_shuffled{
_mm_shufflelo_epi16(b.raw,
(((
3
) << 6) | ((
0
) << 4) | ((
3
) << 2) | (
0
))
)};
return Combine(d, b_shuffled, a_shuffled);
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ShuffleTwo3012(const Vec128<T> a, const Vec128<T> b) {
const DFromV<decltype(a)> d;
const RebindToFloat<decltype(d)> df;
constexpr int m =
(((
3
) << 6) | ((
0
) << 4) | ((
1
) << 2) | (
2
))
;
return BitCast(d, Vec128<float>{_mm_shuffle_ps(BitCast(df, a).raw,
BitCast(df, b).raw, m)});
}
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> Shuffle1032(const Vec128<uint32_t> v) {
return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x4E)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> Shuffle1032(const Vec128<int32_t> v) {
return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x4E)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> Shuffle1032(const Vec128<float> v) {
return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x4E)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t> Shuffle01(const Vec128<uint64_t> v) {
return Vec128<uint64_t>{_mm_shuffle_epi32(v.raw, 0x4E)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t> Shuffle01(const Vec128<int64_t> v) {
return Vec128<int64_t>{_mm_shuffle_epi32(v.raw, 0x4E)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> Shuffle01(const Vec128<double> v) {
return Vec128<double>{_mm_shuffle_pd(v.raw, v.raw, 1)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> Shuffle0321(const Vec128<uint32_t> v) {
return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x39)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> Shuffle0321(const Vec128<int32_t> v) {
return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x39)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> Shuffle0321(const Vec128<float> v) {
return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x39)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> Shuffle2103(const Vec128<uint32_t> v) {
return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x93)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> Shuffle2103(const Vec128<int32_t> v) {
return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x93)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> Shuffle2103(const Vec128<float> v) {
return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x93)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> Shuffle0123(const Vec128<uint32_t> v) {
return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x1B)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> Shuffle0123(const Vec128<int32_t> v) {
return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x1B)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> Shuffle0123(const Vec128<float> v) {
return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x1B)};
}
template <class DTo, typename TFrom, size_t NFrom>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<DTo> RebindMask(DTo dto, Mask128<TFrom, NFrom> m) {
static_assert(sizeof(TFrom) == sizeof(TFromD<DTo>), "Must have same size");
const Simd<TFrom, NFrom, 0> d;
return MaskFromVec(BitCast(dto, VecFromMask(d, m)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> TestBit(Vec128<T, N> v, Vec128<T, N> bit) {
static_assert(!hwy::IsFloat<T>(), "Only integer vectors supported");
return (v & bit) == bit;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint8_t, N> operator==(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Mask128<uint8_t, N>{_mm_cmpeq_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint16_t, N> operator==(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Mask128<uint16_t, N>{_mm_cmpeq_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint32_t, N> operator==(const Vec128<uint32_t, N> a,
const Vec128<uint32_t, N> b) {
return Mask128<uint32_t, N>{_mm_cmpeq_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint64_t, N> operator==(const Vec128<uint64_t, N> a,
const Vec128<uint64_t, N> b) {
const DFromV<decltype(a)> d64;
const RepartitionToNarrow<decltype(d64)> d32;
const auto cmp32 = VecFromMask(d32, Eq(BitCast(d32, a), BitCast(d32, b)));
const auto cmp64 = cmp32 & Shuffle2301(cmp32);
return MaskFromVec(BitCast(d64, cmp64));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int8_t, N> operator==(const Vec128<int8_t, N> a,
const Vec128<int8_t, N> b) {
return Mask128<int8_t, N>{_mm_cmpeq_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int16_t, N> operator==(Vec128<int16_t, N> a,
Vec128<int16_t, N> b) {
return Mask128<int16_t, N>{_mm_cmpeq_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int32_t, N> operator==(const Vec128<int32_t, N> a,
const Vec128<int32_t, N> b) {
return Mask128<int32_t, N>{_mm_cmpeq_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int64_t, N> operator==(const Vec128<int64_t, N> a,
const Vec128<int64_t, N> b) {
const DFromV<decltype(a)> d;
RebindToUnsigned<decltype(d)> du;
return RebindMask(d, BitCast(du, a) == BitCast(du, b));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<float, N> operator==(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Mask128<float, N>{_mm_cmpeq_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<double, N> operator==(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Mask128<double, N>{_mm_cmpeq_pd(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint8_t, N> operator!=(Vec128<uint8_t, N> a,
Vec128<uint8_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint16_t, N> operator!=(Vec128<uint16_t, N> a,
Vec128<uint16_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint32_t, N> operator!=(Vec128<uint32_t, N> a,
Vec128<uint32_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint64_t, N> operator!=(Vec128<uint64_t, N> a,
Vec128<uint64_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int8_t, N> operator!=(Vec128<int8_t, N> a,
Vec128<int8_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int16_t, N> operator!=(Vec128<int16_t, N> a,
Vec128<int16_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int32_t, N> operator!=(Vec128<int32_t, N> a,
Vec128<int32_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int64_t, N> operator!=(Vec128<int64_t, N> a,
Vec128<int64_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<float, N> operator!=(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Mask128<float, N>{_mm_cmpneq_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<double, N> operator!=(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Mask128<double, N>{_mm_cmpneq_pd(a.raw, b.raw)};
}
namespace detail {
template <size_t N>
inline __attribute__((always_inline)) Mask128<int8_t, N> Gt(hwy::SignedTag , Vec128<int8_t, N> a,
Vec128<int8_t, N> b) {
return Mask128<int8_t, N>{_mm_cmpgt_epi8(a.raw, b.raw)};
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<int16_t, N> Gt(hwy::SignedTag , Vec128<int16_t, N> a,
Vec128<int16_t, N> b) {
return Mask128<int16_t, N>{_mm_cmpgt_epi16(a.raw, b.raw)};
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<int32_t, N> Gt(hwy::SignedTag , Vec128<int32_t, N> a,
Vec128<int32_t, N> b) {
return Mask128<int32_t, N>{_mm_cmpgt_epi32(a.raw, b.raw)};
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<int64_t, N> Gt(hwy::SignedTag ,
const Vec128<int64_t, N> a,
const Vec128<int64_t, N> b) {
const DFromV<decltype(a)> d;
const RepartitionToNarrow<decltype(d)> d32;
const Vec128<int64_t, N> m_eq32{Eq(BitCast(d32, a), BitCast(d32, b)).raw};
const Vec128<int64_t, N> m_gt32{Gt(BitCast(d32, a), BitCast(d32, b)).raw};
const __m128i upper = OrAnd(m_gt32, m_eq32, Sub(b, a)).raw;
return Mask128<int64_t, N>{_mm_shuffle_epi32(upper,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Mask128<T, N> Gt(hwy::UnsignedTag , Vec128<T, N> a,
Vec128<T, N> b) {
const DFromV<decltype(a)> du;
const RebindToSigned<decltype(du)> di;
const Vec128<T, N> msb = Set(du, (LimitsMax<T>() >> 1) + 1);
const auto sa = BitCast(di, Xor(a, msb));
const auto sb = BitCast(di, Xor(b, msb));
return RebindMask(du, Gt(hwy::SignedTag(), sa, sb));
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<float, N> Gt(hwy::FloatTag , Vec128<float, N> a,
Vec128<float, N> b) {
return Mask128<float, N>{_mm_cmpgt_ps(a.raw, b.raw)};
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<double, N> Gt(hwy::FloatTag , Vec128<double, N> a,
Vec128<double, N> b) {
return Mask128<double, N>{_mm_cmpgt_pd(a.raw, b.raw)};
}
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Mask128<T, N> operator>(Vec128<T, N> a, Vec128<T, N> b) {
return detail::Gt(hwy::TypeTag<T>(), a, b);
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) Mask128<T, N> Ge(hwy::SignedTag tag, Vec128<T, N> a,
Vec128<T, N> b) {
return Not(Gt(tag, b, a));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Mask128<T, N> Ge(hwy::UnsignedTag tag, Vec128<T, N> a,
Vec128<T, N> b) {
return Not(Gt(tag, b, a));
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<float, N> Ge(hwy::FloatTag , Vec128<float, N> a,
Vec128<float, N> b) {
return Mask128<float, N>{_mm_cmpge_ps(a.raw, b.raw)};
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<double, N> Ge(hwy::FloatTag , Vec128<double, N> a,
Vec128<double, N> b) {
return Mask128<double, N>{_mm_cmpge_pd(a.raw, b.raw)};
}
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> operator>=(Vec128<T, N> a, Vec128<T, N> b) {
return detail::Ge(hwy::TypeTag<T>(), a, b);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> operator<(Vec128<T, N> a, Vec128<T, N> b) {
return b > a;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> operator<=(Vec128<T, N> a, Vec128<T, N> b) {
return b >= a;
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{_mm_set_epi8(
static_cast<char>(15), static_cast<char>(14), static_cast<char>(13),
static_cast<char>(12), static_cast<char>(11), static_cast<char>(10),
static_cast<char>(9), static_cast<char>(8), static_cast<char>(7),
static_cast<char>(6), static_cast<char>(5), static_cast<char>(4),
static_cast<char>(3), static_cast<char>(2), static_cast<char>(1),
static_cast<char>(0))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr,
hwy::EnableIf<!hwy::IsSpecialFloat<TFromD<D> >()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{_mm_set_epi16(int16_t{7}, int16_t{6}, int16_t{5}, int16_t{4},
int16_t{3}, int16_t{2}, int16_t{1},
int16_t{0})};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>() || IsSame<TFromD<D>, int32_t>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{
_mm_set_epi32(int32_t{3}, int32_t{2}, int32_t{1}, int32_t{0})};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>() || IsSame<TFromD<D>, int64_t>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{_mm_set_epi64x(int64_t{1}, int64_t{0})};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{_mm_set_ps(3.0f, 2.0f, 1.0f, 0.0f)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{_mm_set_pd(1.0, 0.0)};
}
}
template <class D, typename T2, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Iota(D d, const T2 first) {
const auto result_iota =
detail::Iota0(d) + Set(d, static_cast<TFromD<D>>(first));
return result_iota;
}
template <class D, class M = MFromD<D>, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) M FirstN(D d, size_t num) {
const RebindToSigned<decltype(d)> di;
using TI = TFromD<decltype(di)>;
return RebindMask(d, detail::Iota0(di) < Set(di, static_cast<TI>(num)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> MaskedLoad(MFromD<D> m, D d, const T* __restrict__ p) {
return IfThenElseZero(m, LoadU(d, p));
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> MaskedLoadOr(VFromD<D> v, MFromD<D> m, D d,
const T* __restrict__ p) {
return IfThenElse(m, LoadU(d, p), v);
}
namespace detail {
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void ScalarMaskedStore(VFromD<D> v, MFromD<D> m, D d,
TFromD<D>* __restrict__ p) {
const RebindToSigned<decltype(d)> di;
using TI = TFromD<decltype(di)>;
alignas(16) TI buf[MaxLanes(d)];
alignas(16) TI mask[MaxLanes(d)];
Store(BitCast(di, v), di, buf);
Store(BitCast(di, VecFromMask(d, m)), di, mask);
for (size_t i = 0; i < MaxLanes(d); ++i) {
if (mask[i]) {
CopySameSize(buf + i, p + i);
}
}
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void BlendedStore(VFromD<D> v, MFromD<D> m, D d,
TFromD<D>* __restrict__ p) {
detail::ScalarMaskedStore(v, m, d, p);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> operator+(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_add_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> operator+(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_add_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> operator+(const Vec128<uint32_t, N> a,
const Vec128<uint32_t, N> b) {
return Vec128<uint32_t, N>{_mm_add_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> operator+(const Vec128<uint64_t, N> a,
const Vec128<uint64_t, N> b) {
return Vec128<uint64_t, N>{_mm_add_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> operator+(const Vec128<int8_t, N> a,
const Vec128<int8_t, N> b) {
return Vec128<int8_t, N>{_mm_add_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> operator+(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_add_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> operator+(const Vec128<int32_t, N> a,
const Vec128<int32_t, N> b) {
return Vec128<int32_t, N>{_mm_add_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> operator+(const Vec128<int64_t, N> a,
const Vec128<int64_t, N> b) {
return Vec128<int64_t, N>{_mm_add_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> operator+(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_add_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> operator+(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_add_pd(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> operator-(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_sub_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> operator-(Vec128<uint16_t, N> a,
Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_sub_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> operator-(const Vec128<uint32_t, N> a,
const Vec128<uint32_t, N> b) {
return Vec128<uint32_t, N>{_mm_sub_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> operator-(const Vec128<uint64_t, N> a,
const Vec128<uint64_t, N> b) {
return Vec128<uint64_t, N>{_mm_sub_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> operator-(const Vec128<int8_t, N> a,
const Vec128<int8_t, N> b) {
return Vec128<int8_t, N>{_mm_sub_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> operator-(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_sub_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> operator-(const Vec128<int32_t, N> a,
const Vec128<int32_t, N> b) {
return Vec128<int32_t, N>{_mm_sub_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> operator-(const Vec128<int64_t, N> a,
const Vec128<int64_t, N> b) {
return Vec128<int64_t, N>{_mm_sub_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> operator-(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_sub_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> operator-(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_sub_pd(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N / 8> SumsOf8(const Vec128<uint8_t, N> v) {
return Vec128<uint64_t, N / 8>{_mm_sad_epu8(v.raw, _mm_setzero_si128())};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N / 8> SumsOf8AbsDiff(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint64_t, N / 8>{_mm_sad_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> SaturatedAdd(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_adds_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> SaturatedAdd(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_adds_epu16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> SaturatedAdd(const Vec128<int8_t, N> a,
const Vec128<int8_t, N> b) {
return Vec128<int8_t, N>{_mm_adds_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> SaturatedAdd(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_adds_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> SaturatedSub(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_subs_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> SaturatedSub(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_subs_epu16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> SaturatedSub(const Vec128<int8_t, N> a,
const Vec128<int8_t, N> b) {
return Vec128<int8_t, N>{_mm_subs_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> SaturatedSub(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_subs_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> AverageRound(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_avg_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> AverageRound(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_avg_epu16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> operator*(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_mullo_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> operator*(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_mullo_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> MulHigh(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_mulhi_epu16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> MulHigh(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_mulhi_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, (N + 1) / 2> MulEven(const Vec128<uint32_t, N> a,
const Vec128<uint32_t, N> b) {
return Vec128<uint64_t, (N + 1) / 2>{_mm_mul_epu32(a.raw, b.raw)};
}
template <size_t N, hwy::EnableIf<N * sizeof(int32_t) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, (N + 1) / 2> MulEven(const Vec128<int32_t, N> a,
const Vec128<int32_t, N> b) {
const DFromV<decltype(a)> d;
const RepartitionToWide<decltype(d)> dw;
return Set(dw, static_cast<int64_t>(GetLane(a)) * GetLane(b));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t> MulEven(Vec128<int32_t> a, Vec128<int32_t> b) {
alignas(16) int32_t a_lanes[4];
alignas(16) int32_t b_lanes[4];
const DFromV<decltype(a)> di32;
const RepartitionToWide<decltype(di32)> di64;
Store(a, di32, a_lanes);
Store(b, di32, b_lanes);
alignas(16) int64_t mul[2];
mul[0] = static_cast<int64_t>(a_lanes[0]) * b_lanes[0];
mul[1] = static_cast<int64_t>(a_lanes[2]) * b_lanes[2];
return Load(di64, mul);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> operator*(const Vec128<uint32_t, N> a,
const Vec128<uint32_t, N> b) {
const __m128i a_x3x1 = _mm_shuffle_epi32(a.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
);
const auto mullo_x2x0 = MulEven(a, b);
const __m128i b_x3x1 = _mm_shuffle_epi32(b.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
);
const auto mullo_x3x1 =
MulEven(Vec128<uint32_t, N>{a_x3x1}, Vec128<uint32_t, N>{b_x3x1});
const __m128i mul_20 =
_mm_shuffle_epi32(mullo_x2x0.raw,
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
);
const __m128i mul_31 =
_mm_shuffle_epi32(mullo_x3x1.raw,
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
);
return Vec128<uint32_t, N>{_mm_unpacklo_epi32(mul_20, mul_31)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> operator*(const Vec128<int32_t, N> a,
const Vec128<int32_t, N> b) {
const DFromV<decltype(a)> d;
const RebindToUnsigned<decltype(d)> du;
return BitCast(d, BitCast(du, a) * BitCast(du, b));
}
template <int kBits, typename T, size_t N,
hwy::EnableIf<((size_t{1} << sizeof(T)) & ((1 << 1) | (1 << 2))) != 0>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> RotateRight(const Vec128<T, N> v) {
constexpr size_t kSizeInBits = sizeof(T) * 8;
static_assert(0 <= kBits && kBits < kSizeInBits, "Invalid shift count");
if (kBits == 0) return v;
return Or(ShiftRight<kBits>(v),
ShiftLeft<((kSizeInBits - 1) < (kSizeInBits - kBits) ? (kSizeInBits - 1) : (kSizeInBits - kBits))>(v));
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> RotateRight(const Vec128<uint32_t, N> v) {
static_assert(0 <= kBits && kBits < 32, "Invalid shift count");
if (kBits == 0) return v;
return Or(ShiftRight<kBits>(v), ShiftLeft<((31) < (32 - kBits) ? (31) : (32 - kBits))>(v));
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> RotateRight(const Vec128<uint64_t, N> v) {
static_assert(0 <= kBits && kBits < 64, "Invalid shift count");
if (kBits == 0) return v;
return Or(ShiftRight<kBits>(v), ShiftLeft<((63) < (64 - kBits) ? (63) : (64 - kBits))>(v));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> BroadcastSignBit(const Vec128<int8_t, N> v) {
const DFromV<decltype(v)> d;
return VecFromMask(v < Zero(d));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> BroadcastSignBit(const Vec128<int16_t, N> v) {
return ShiftRight<15>(v);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> BroadcastSignBit(const Vec128<int32_t, N> v) {
return ShiftRight<31>(v);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> BroadcastSignBit(const Vec128<int64_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToNarrow<decltype(d)> d32;
const auto sign = ShiftRight<31>(BitCast(d32, v));
return Vec128<int64_t, N>{
_mm_shuffle_epi32(sign.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> Abs(const Vec128<int8_t, N> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const auto zero = Zero(du);
const auto v_as_u8 = BitCast(du, v);
return BitCast(d, Min(v_as_u8, zero - v_as_u8));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> Abs(const Vec128<int16_t, N> v) {
const auto zero = Zero(DFromV<decltype(v)>());
return Max(v, zero - v);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> Abs(const Vec128<int32_t, N> v) {
const auto zero = Zero(DFromV<decltype(v)>());
return IfThenElse(MaskFromVec(BroadcastSignBit(v)), zero - v, v);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> Abs(const Vec128<int64_t, N> v) {
const auto zero = Zero(DFromV<decltype(v)>());
return IfThenElse(MaskFromVec(BroadcastSignBit(v)), zero - v, v);
}
using Shift64Count = int;
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> ShiftRight(const Vec128<int64_t, N> v) {
const DFromV<decltype(v)> di;
const RebindToUnsigned<decltype(di)> du;
const auto right = BitCast(di, ShiftRight<kBits>(BitCast(du, v)));
const auto sign = ShiftLeft<64 - kBits>(BroadcastSignBit(v));
return right | sign;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ZeroIfNegative(Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only works for float");
const DFromV<decltype(v)> d;
const RebindToSigned<decltype(d)> di;
const auto mask = MaskFromVec(BitCast(d, BroadcastSignBit(BitCast(di, v))));
return IfThenElse(mask, Zero(d), v);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> IfNegativeThenElse(const Vec128<int8_t, N> v,
const Vec128<int8_t, N> yes,
const Vec128<int8_t, N> no) {
return IfThenElse(MaskFromVec(v), yes, no);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfNegativeThenElse(Vec128<T, N> v, Vec128<T, N> yes,
Vec128<T, N> no) {
static_assert(IsSigned<T>(), "Only works for signed/float");
const DFromV<decltype(v)> d;
const RebindToSigned<decltype(d)> di;
v = BitCast(d, BroadcastSignBit(BitCast(di, v)));
return IfThenElse(MaskFromVec(v), yes, no);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) != (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfNegativeThenElse(Vec128<T, N> v, Vec128<T, N> yes,
Vec128<T, N> no) {
static_assert(IsSigned<T>(), "Only works for signed/float");
const DFromV<decltype(v)> d;
const RebindToFloat<decltype(d)> df;
return BitCast(d, IfThenElse(MaskFromVec(BitCast(df, v)), BitCast(df, yes),
BitCast(df, no)));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> ShiftLeftSame(const Vec128<uint16_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint16_t, N>{_mm_slli_epi16(v.raw, bits)};
}
return Vec128<uint16_t, N>{_mm_sll_epi16(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> ShiftLeftSame(const Vec128<uint32_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint32_t, N>{_mm_slli_epi32(v.raw, bits)};
}
return Vec128<uint32_t, N>{_mm_sll_epi32(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> ShiftLeftSame(const Vec128<uint64_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint64_t, N>{_mm_slli_epi64(v.raw, bits)};
}
return Vec128<uint64_t, N>{_mm_sll_epi64(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> ShiftLeftSame(const Vec128<int16_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<int16_t, N>{_mm_slli_epi16(v.raw, bits)};
}
return Vec128<int16_t, N>{_mm_sll_epi16(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> ShiftLeftSame(const Vec128<int32_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<int32_t, N>{_mm_slli_epi32(v.raw, bits)};
}
return Vec128<int32_t, N>{_mm_sll_epi32(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> ShiftLeftSame(const Vec128<int64_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<int64_t, N>{_mm_slli_epi64(v.raw, bits)};
}
return Vec128<int64_t, N>{_mm_sll_epi64(v.raw, _mm_cvtsi32_si128(bits))};
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftLeftSame(const Vec128<T, N> v, const int bits) {
const DFromV<decltype(v)> d8;
const Vec128<T, N> shifted{
ShiftLeftSame(Vec128<MakeWide<T>>{v.raw}, bits).raw};
return shifted & Set(d8, static_cast<T>((0xFF << bits) & 0xFF));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> ShiftRightSame(const Vec128<uint16_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint16_t, N>{_mm_srli_epi16(v.raw, bits)};
}
return Vec128<uint16_t, N>{_mm_srl_epi16(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> ShiftRightSame(const Vec128<uint32_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint32_t, N>{_mm_srli_epi32(v.raw, bits)};
}
return Vec128<uint32_t, N>{_mm_srl_epi32(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> ShiftRightSame(const Vec128<uint64_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint64_t, N>{_mm_srli_epi64(v.raw, bits)};
}
return Vec128<uint64_t, N>{_mm_srl_epi64(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> ShiftRightSame(Vec128<uint8_t, N> v,
const int bits) {
const DFromV<decltype(v)> d8;
const Vec128<uint8_t, N> shifted{
ShiftRightSame(Vec128<uint16_t>{v.raw}, bits).raw};
return shifted & Set(d8, static_cast<uint8_t>(0xFF >> bits));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> ShiftRightSame(const Vec128<int16_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<int16_t, N>{_mm_srai_epi16(v.raw, bits)};
}
return Vec128<int16_t, N>{_mm_sra_epi16(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> ShiftRightSame(const Vec128<int32_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<int32_t, N>{_mm_srai_epi32(v.raw, bits)};
}
return Vec128<int32_t, N>{_mm_sra_epi32(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> ShiftRightSame(const Vec128<int64_t, N> v,
const int bits) {
const DFromV<decltype(v)> di;
const RebindToUnsigned<decltype(di)> du;
const auto right = BitCast(di, ShiftRightSame(BitCast(du, v), bits));
const auto sign = ShiftLeftSame(BroadcastSignBit(v), 64 - bits);
return right | sign;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> ShiftRightSame(Vec128<int8_t, N> v, const int bits) {
const DFromV<decltype(v)> di;
const RebindToUnsigned<decltype(di)> du;
const auto shifted = BitCast(di, ShiftRightSame(BitCast(du, v), bits));
const auto shifted_sign =
BitCast(di, Set(du, static_cast<uint8_t>(0x80 >> bits)));
return (shifted ^ shifted_sign) - shifted_sign;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> operator*(Vec128<float, N> a, Vec128<float, N> b) {
return Vec128<float, N>{_mm_mul_ps(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, 1> operator*(const Vec128<float, 1> a,
const Vec128<float, 1> b) {
return Vec128<float, 1>{_mm_mul_ss(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> operator*(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_mul_pd(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<double> operator*(const Vec64<double> a, const Vec64<double> b) {
return Vec64<double>{_mm_mul_sd(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> operator/(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_div_ps(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, 1> operator/(const Vec128<float, 1> a,
const Vec128<float, 1> b) {
return Vec128<float, 1>{_mm_div_ss(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> operator/(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_div_pd(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<double> operator/(const Vec64<double> a, const Vec64<double> b) {
return Vec64<double>{_mm_div_sd(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> ApproximateReciprocal(const Vec128<float, N> v) {
return Vec128<float, N>{_mm_rcp_ps(v.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, 1> ApproximateReciprocal(const Vec128<float, 1> v) {
return Vec128<float, 1>{_mm_rcp_ss(v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> AbsDiff(Vec128<float, N> a, Vec128<float, N> b) {
return Abs(a - b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> MulAdd(Vec128<float, N> mul, Vec128<float, N> x,
Vec128<float, N> add) {
return mul * x + add;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> MulAdd(Vec128<double, N> mul, Vec128<double, N> x,
Vec128<double, N> add) {
return mul * x + add;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> NegMulAdd(Vec128<float, N> mul, Vec128<float, N> x,
Vec128<float, N> add) {
return add - mul * x;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> NegMulAdd(Vec128<double, N> mul, Vec128<double, N> x,
Vec128<double, N> add) {
return add - mul * x;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> MulSub(Vec128<float, N> mul, Vec128<float, N> x,
Vec128<float, N> sub) {
return mul * x - sub;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> MulSub(Vec128<double, N> mul, Vec128<double, N> x,
Vec128<double, N> sub) {
return mul * x - sub;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> NegMulSub(Vec128<float, N> mul, Vec128<float, N> x,
Vec128<float, N> sub) {
return Neg(mul) * x - sub;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> NegMulSub(Vec128<double, N> mul, Vec128<double, N> x,
Vec128<double, N> sub) {
return Neg(mul) * x - sub;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Sqrt(Vec128<float, N> v) {
return Vec128<float, N>{_mm_sqrt_ps(v.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, 1> Sqrt(Vec128<float, 1> v) {
return Vec128<float, 1>{_mm_sqrt_ss(v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Sqrt(Vec128<double, N> v) {
return Vec128<double, N>{_mm_sqrt_pd(v.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<double> Sqrt(Vec64<double> v) {
return Vec64<double>{_mm_sqrt_sd(_mm_setzero_pd(), v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> ApproximateReciprocalSqrt(Vec128<float, N> v) {
return Vec128<float, N>{_mm_rsqrt_ps(v.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, 1> ApproximateReciprocalSqrt(Vec128<float, 1> v) {
return Vec128<float, 1>{_mm_rsqrt_ss(v.raw)};
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) __attribute__((unused)) Vec128<T, N> MinU(const Vec128<T, N> a,
const Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const RebindToUnsigned<decltype(d)> du;
const RebindToSigned<decltype(d)> di;
const auto msb = Set(du, static_cast<T>(T(1) << (sizeof(T) * 8 - 1)));
const auto gt = RebindMask(du, BitCast(di, a ^ msb) > BitCast(di, b ^ msb));
return IfThenElse(gt, b, a);
}
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> Min(Vec128<uint8_t, N> a, Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_min_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> Min(Vec128<uint16_t, N> a, Vec128<uint16_t, N> b) {
return detail::MinU(a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> Min(Vec128<uint32_t, N> a, Vec128<uint32_t, N> b) {
return detail::MinU(a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> Min(Vec128<uint64_t, N> a, Vec128<uint64_t, N> b) {
return detail::MinU(a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> Min(Vec128<int8_t, N> a, Vec128<int8_t, N> b) {
return IfThenElse(a < b, a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> Min(Vec128<int16_t, N> a, Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_min_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> Min(Vec128<int32_t, N> a, Vec128<int32_t, N> b) {
return IfThenElse(a < b, a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> Min(Vec128<int64_t, N> a, Vec128<int64_t, N> b) {
return IfThenElse(a < b, a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Min(Vec128<float, N> a, Vec128<float, N> b) {
return Vec128<float, N>{_mm_min_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Min(Vec128<double, N> a, Vec128<double, N> b) {
return Vec128<double, N>{_mm_min_pd(a.raw, b.raw)};
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) __attribute__((unused)) Vec128<T, N> MaxU(const Vec128<T, N> a,
const Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const RebindToUnsigned<decltype(d)> du;
const RebindToSigned<decltype(d)> di;
const auto msb = Set(du, static_cast<T>(T(1) << (sizeof(T) * 8 - 1)));
const auto gt = RebindMask(du, BitCast(di, a ^ msb) > BitCast(di, b ^ msb));
return IfThenElse(gt, a, b);
}
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> Max(Vec128<uint8_t, N> a, Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_max_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> Max(Vec128<uint16_t, N> a, Vec128<uint16_t, N> b) {
return detail::MaxU(a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> Max(Vec128<uint32_t, N> a, Vec128<uint32_t, N> b) {
return detail::MaxU(a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> Max(Vec128<uint64_t, N> a, Vec128<uint64_t, N> b) {
return detail::MaxU(a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> Max(Vec128<int8_t, N> a, Vec128<int8_t, N> b) {
return IfThenElse(a < b, b, a);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> Max(Vec128<int16_t, N> a, Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_max_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> Max(Vec128<int32_t, N> a, Vec128<int32_t, N> b) {
return IfThenElse(a < b, b, a);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> Max(Vec128<int64_t, N> a, Vec128<int64_t, N> b) {
return IfThenElse(a < b, b, a);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Max(Vec128<float, N> a, Vec128<float, N> b) {
return Vec128<float, N>{_mm_max_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Max(Vec128<double, N> a, Vec128<double, N> b) {
return Vec128<double, N>{_mm_max_pd(a.raw, b.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<!hwy::IsFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Stream(VFromD<D> v, D , TFromD<D>* __restrict__ aligned) {
_mm_stream_si128(reinterpret_cast<__m128i*>(aligned), v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Stream(VFromD<D> v, D , float* __restrict__ aligned) {
_mm_stream_ps(aligned, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Stream(VFromD<D> v, D , double* __restrict__ aligned) {
_mm_stream_pd(aligned, v.raw);
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-conversion"
using GatherIndex64 = long long int;
static_assert(sizeof(GatherIndex64) == 8, "Must be 64-bit type");
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>, class VI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void ScatterOffset(VFromD<D> v, D d, T* __restrict__ base, VI offset) {
using TI = TFromV<VI>;
static_assert(sizeof(T) == sizeof(TI), "Index/lane size must match");
alignas(16) T lanes[MaxLanes(d)];
Store(v, d, lanes);
alignas(16) TI offset_lanes[MaxLanes(d)];
Store(offset, Rebind<TI, decltype(d)>(), offset_lanes);
uint8_t* base_bytes = reinterpret_cast<uint8_t*>(base);
for (size_t i = 0; i < MaxLanes(d); ++i) {
CopyBytes<sizeof(T)>(&lanes[i], base_bytes + offset_lanes[i]);
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>, class VI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void ScatterIndex(VFromD<D> v, D d, T* __restrict__ base, VI index) {
using TI = TFromV<VI>;
static_assert(sizeof(T) == sizeof(TI), "Index/lane size must match");
alignas(16) T lanes[MaxLanes(d)];
Store(v, d, lanes);
alignas(16) TI index_lanes[MaxLanes(d)];
Store(index, Rebind<TI, decltype(d)>(), index_lanes);
for (size_t i = 0; i < MaxLanes(d); ++i) {
base[index_lanes[i]] = lanes[i];
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>, class VI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> GatherOffset(D d, const T* __restrict__ base, VI offset) {
using TI = TFromV<VI>;
static_assert(sizeof(T) == sizeof(TI), "Index/lane size must match");
alignas(16) TI offset_lanes[MaxLanes(d)];
Store(offset, Rebind<TI, decltype(d)>(), offset_lanes);
alignas(16) T lanes[MaxLanes(d)];
const uint8_t* base_bytes = reinterpret_cast<const uint8_t*>(base);
for (size_t i = 0; i < MaxLanes(d); ++i) {
CopyBytes<sizeof(T)>(base_bytes + offset_lanes[i], &lanes[i]);
}
return Load(d, lanes);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>, class VI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> GatherIndex(D d, const T* __restrict__ base, VI index) {
using TI = TFromV<VI>;
static_assert(sizeof(T) == sizeof(TI), "Index/lane size must match");
alignas(16) TI index_lanes[MaxLanes(d)];
Store(index, Rebind<TI, decltype(d)>(), index_lanes);
alignas(16) T lanes[MaxLanes(d)];
for (size_t i = 0; i < MaxLanes(d); ++i) {
lanes[i] = base[index_lanes[i]];
}
return Load(d, lanes);
}
#pragma GCC diagnostic pop
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> LowerHalf(D , VFromD<Twice<D>> v) {
return VFromD<D>{v.raw};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N / 2> LowerHalf(Vec128<T, N> v) {
return Vec128<T, N / 2>{v.raw};
}
template <int kBytes, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ShiftLeftBytes(D d, VFromD<D> v) {
static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes");
const RebindToUnsigned<decltype(d)> du;
return BitCast(
d, VFromD<decltype(du)>{_mm_slli_si128(BitCast(du, v).raw, kBytes)});
}
template <int kBytes, typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftLeftBytes(const Vec128<T, N> v) {
return ShiftLeftBytes<kBytes>(DFromV<decltype(v)>(), v);
}
template <int kLanes, class D, typename T = TFromD<D>,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ShiftLeftLanes(D d, const VFromD<D> v) {
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, ShiftLeftBytes<kLanes * sizeof(T)>(BitCast(d8, v)));
}
template <int kLanes, typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftLeftLanes(const Vec128<T, N> v) {
return ShiftLeftLanes<kLanes>(DFromV<decltype(v)>(), v);
}
template <int kBytes, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ShiftRightBytes(D d, VFromD<D> v) {
static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes");
const RebindToUnsigned<decltype(d)> du;
if (d.MaxBytes() != 16) {
const Full128<TFromD<D>> dfull;
const VFromD<decltype(dfull)> vfull{v.raw};
v = VFromD<D>{IfThenElseZero(FirstN(dfull, MaxLanes(d)), vfull).raw};
}
return BitCast(
d, VFromD<decltype(du)>{_mm_srli_si128(BitCast(du, v).raw, kBytes)});
}
template <int kLanes, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ShiftRightLanes(D d, const VFromD<D> v) {
const Repartition<uint8_t, decltype(d)> d8;
constexpr size_t kBytes = kLanes * sizeof(TFromD<D>);
return BitCast(d, ShiftRightBytes<kBytes>(d8, BitCast(d8, v)));
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> UpperHalf(D , Vec128<T> v) {
return Vec64<T>{_mm_unpackhi_epi64(v.raw, v.raw)};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<float> UpperHalf(D , Vec128<float> v) {
return Vec64<float>{_mm_movehl_ps(v.raw, v.raw)};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<double> UpperHalf(D , Vec128<double> v) {
return Vec64<double>{_mm_unpackhi_pd(v.raw, v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> UpperHalf(D d, VFromD<Twice<D>> v) {
return LowerHalf(d, ShiftRightBytes<d.MaxBytes()>(Twice<D>(), v));
}
namespace detail {
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
inline __attribute__((always_inline)) T ExtractLane(const Vec128<T, N> v) {
static_assert(kLane < N, "Lane index out of bounds");
const int pair = _mm_extract_epi16(v.raw, kLane / 2);
constexpr int kShift = kLane & 1 ? 8 : 0;
return static_cast<T>((pair >> kShift) & 0xFF);
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
inline __attribute__((always_inline)) T ExtractLane(const Vec128<T, N> v) {
static_assert(kLane < N, "Lane index out of bounds");
return static_cast<T>(_mm_extract_epi16(v.raw, kLane) & 0xFFFF);
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) T ExtractLane(const Vec128<T, N> v) {
static_assert(kLane < N, "Lane index out of bounds");
alignas(16) T lanes[4];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[kLane];
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) T ExtractLane(const Vec128<T, N> v) {
static_assert(kLane < N, "Lane index out of bounds");
alignas(16) T lanes[2];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[kLane];
}
template <size_t kLane, size_t N>
inline __attribute__((always_inline)) float ExtractLane(const Vec128<float, N> v) {
static_assert(kLane < N, "Lane index out of bounds");
alignas(16) float lanes[4];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[kLane];
}
template <size_t kLane>
inline __attribute__((always_inline)) double ExtractLane(const Vec128<double, 1> v) {
static_assert(kLane == 0, "Lane index out of bounds");
return GetLane(v);
}
template <size_t kLane>
inline __attribute__((always_inline)) double ExtractLane(const Vec128<double> v) {
static_assert(kLane < 2, "Lane index out of bounds");
const Half<DFromV<decltype(v)>> dh;
return kLane == 0 ? GetLane(v) : GetLane(UpperHalf(dh, v));
}
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T ExtractLane(const Vec128<T, 1> v, size_t i) {
do { } while (0);
(void)i;
return GetLane(v);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T ExtractLane(const Vec128<T, 2> v, size_t i) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::ExtractLane<0>(v);
case 1:
return detail::ExtractLane<1>(v);
}
}
alignas(16) T lanes[2];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[i];
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T ExtractLane(const Vec128<T, 4> v, size_t i) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::ExtractLane<0>(v);
case 1:
return detail::ExtractLane<1>(v);
case 2:
return detail::ExtractLane<2>(v);
case 3:
return detail::ExtractLane<3>(v);
}
}
alignas(16) T lanes[4];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[i];
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T ExtractLane(const Vec128<T, 8> v, size_t i) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::ExtractLane<0>(v);
case 1:
return detail::ExtractLane<1>(v);
case 2:
return detail::ExtractLane<2>(v);
case 3:
return detail::ExtractLane<3>(v);
case 4:
return detail::ExtractLane<4>(v);
case 5:
return detail::ExtractLane<5>(v);
case 6:
return detail::ExtractLane<6>(v);
case 7:
return detail::ExtractLane<7>(v);
}
}
alignas(16) T lanes[8];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[i];
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T ExtractLane(const Vec128<T, 16> v, size_t i) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::ExtractLane<0>(v);
case 1:
return detail::ExtractLane<1>(v);
case 2:
return detail::ExtractLane<2>(v);
case 3:
return detail::ExtractLane<3>(v);
case 4:
return detail::ExtractLane<4>(v);
case 5:
return detail::ExtractLane<5>(v);
case 6:
return detail::ExtractLane<6>(v);
case 7:
return detail::ExtractLane<7>(v);
case 8:
return detail::ExtractLane<8>(v);
case 9:
return detail::ExtractLane<9>(v);
case 10:
return detail::ExtractLane<10>(v);
case 11:
return detail::ExtractLane<11>(v);
case 12:
return detail::ExtractLane<12>(v);
case 13:
return detail::ExtractLane<13>(v);
case 14:
return detail::ExtractLane<14>(v);
case 15:
return detail::ExtractLane<15>(v);
}
}
alignas(16) T lanes[16];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[i];
}
namespace detail {
template <class V>
inline __attribute__((always_inline)) V InsertLaneUsingBroadcastAndBlend(V v, size_t i, TFromV<V> t) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
const auto mask = RebindMask(d, Iota(du, 0) == Set(du, static_cast<TU>(i)));
return IfThenElse(mask, Set(d, t), v);
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> InsertLane(const Vec128<T, N> v, T t) {
static_assert(kLane < N, "Lane index out of bounds");
return InsertLaneUsingBroadcastAndBlend(v, kLane, t);
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> InsertLane(const Vec128<T, N> v, T t) {
static_assert(kLane < N, "Lane index out of bounds");
return Vec128<T, N>{_mm_insert_epi16(v.raw, t, kLane)};
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> InsertLane(const Vec128<T, N> v, T t) {
static_assert(kLane < N, "Lane index out of bounds");
return InsertLaneUsingBroadcastAndBlend(v, kLane, t);
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> InsertLane(const Vec128<T, N> v, T t) {
static_assert(kLane < N, "Lane index out of bounds");
const DFromV<decltype(v)> d;
const RebindToFloat<decltype(d)> df;
const auto vt = BitCast(df, Set(d, t));
if (kLane == 0) {
return BitCast(
d, Vec128<double, N>{_mm_shuffle_pd(vt.raw, BitCast(df, v).raw, 2)});
}
return BitCast(
d, Vec128<double, N>{_mm_shuffle_pd(BitCast(df, v).raw, vt.raw, 0)});
}
template <size_t kLane, size_t N>
inline __attribute__((always_inline)) Vec128<float, N> InsertLane(const Vec128<float, N> v, float t) {
static_assert(kLane < N, "Lane index out of bounds");
return InsertLaneUsingBroadcastAndBlend(v, kLane, t);
}
template <size_t kLane>
inline __attribute__((always_inline)) Vec128<double, 1> InsertLane(const Vec128<double, 1> v, double t) {
static_assert(kLane == 0, "Lane index out of bounds");
return Set(DFromV<decltype(v)>(), t);
}
template <size_t kLane>
inline __attribute__((always_inline)) Vec128<double> InsertLane(const Vec128<double> v, double t) {
static_assert(kLane < 2, "Lane index out of bounds");
const DFromV<decltype(v)> d;
const Vec128<double> vt = Set(d, t);
if (kLane == 0) {
return Vec128<double>{_mm_shuffle_pd(vt.raw, v.raw, 2)};
}
return Vec128<double>{_mm_shuffle_pd(v.raw, vt.raw, 0)};
}
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> InsertLane(const Vec128<T, 1> v, size_t i, T t) {
do { } while (0);
(void)i;
return Set(DFromV<decltype(v)>(), t);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 2> InsertLane(const Vec128<T, 2> v, size_t i, T t) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::InsertLane<0>(v, t);
case 1:
return detail::InsertLane<1>(v, t);
}
}
return detail::InsertLaneUsingBroadcastAndBlend(v, i, t);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 4> InsertLane(const Vec128<T, 4> v, size_t i, T t) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::InsertLane<0>(v, t);
case 1:
return detail::InsertLane<1>(v, t);
case 2:
return detail::InsertLane<2>(v, t);
case 3:
return detail::InsertLane<3>(v, t);
}
}
return detail::InsertLaneUsingBroadcastAndBlend(v, i, t);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 8> InsertLane(const Vec128<T, 8> v, size_t i, T t) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::InsertLane<0>(v, t);
case 1:
return detail::InsertLane<1>(v, t);
case 2:
return detail::InsertLane<2>(v, t);
case 3:
return detail::InsertLane<3>(v, t);
case 4:
return detail::InsertLane<4>(v, t);
case 5:
return detail::InsertLane<5>(v, t);
case 6:
return detail::InsertLane<6>(v, t);
case 7:
return detail::InsertLane<7>(v, t);
}
}
return detail::InsertLaneUsingBroadcastAndBlend(v, i, t);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 16> InsertLane(const Vec128<T, 16> v, size_t i, T t) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::InsertLane<0>(v, t);
case 1:
return detail::InsertLane<1>(v, t);
case 2:
return detail::InsertLane<2>(v, t);
case 3:
return detail::InsertLane<3>(v, t);
case 4:
return detail::InsertLane<4>(v, t);
case 5:
return detail::InsertLane<5>(v, t);
case 6:
return detail::InsertLane<6>(v, t);
case 7:
return detail::InsertLane<7>(v, t);
case 8:
return detail::InsertLane<8>(v, t);
case 9:
return detail::InsertLane<9>(v, t);
case 10:
return detail::InsertLane<10>(v, t);
case 11:
return detail::InsertLane<11>(v, t);
case 12:
return detail::InsertLane<12>(v, t);
case 13:
return detail::InsertLane<13>(v, t);
case 14:
return detail::InsertLane<14>(v, t);
case 15:
return detail::InsertLane<15>(v, t);
}
}
return detail::InsertLaneUsingBroadcastAndBlend(v, i, t);
}
template <int kBytes, class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> CombineShiftRightBytes(D d, Vec128<T> hi, Vec128<T> lo) {
static_assert(0 < kBytes && kBytes < 16, "kBytes invalid");
return Or(ShiftRightBytes<kBytes>(d, lo), ShiftLeftBytes<16 - kBytes>(d, hi));
}
template <int kBytes, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> CombineShiftRightBytes(D d, VFromD<D> hi, VFromD<D> lo) {
constexpr size_t kSize = d.MaxBytes();
static_assert(0 < kBytes && kBytes < kSize, "kBytes invalid");
const Twice<decltype(d)> dt;
return VFromD<D>{ShiftRightBytes<kBytes>(dt, Combine(dt, hi, lo)).raw};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> Broadcast(const Vec128<uint16_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
if (kLane < 4) {
const __m128i lo = _mm_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF);
return Vec128<uint16_t, N>{_mm_unpacklo_epi64(lo, lo)};
} else {
const __m128i hi = _mm_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF);
return Vec128<uint16_t, N>{_mm_unpackhi_epi64(hi, hi)};
}
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> Broadcast(const Vec128<uint32_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<uint32_t, N>{_mm_shuffle_epi32(v.raw, 0x55 * kLane)};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> Broadcast(const Vec128<uint64_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<uint64_t, N>{_mm_shuffle_epi32(v.raw, kLane ? 0xEE : 0x44)};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> Broadcast(const Vec128<int16_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
if (kLane < 4) {
const __m128i lo = _mm_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF);
return Vec128<int16_t, N>{_mm_unpacklo_epi64(lo, lo)};
} else {
const __m128i hi = _mm_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF);
return Vec128<int16_t, N>{_mm_unpackhi_epi64(hi, hi)};
}
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> Broadcast(const Vec128<int32_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<int32_t, N>{_mm_shuffle_epi32(v.raw, 0x55 * kLane)};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> Broadcast(const Vec128<int64_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<int64_t, N>{_mm_shuffle_epi32(v.raw, kLane ? 0xEE : 0x44)};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Broadcast(const Vec128<float, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<float, N>{_mm_shuffle_ps(v.raw, v.raw, 0x55 * kLane)};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Broadcast(const Vec128<double, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<double, N>{_mm_shuffle_pd(v.raw, v.raw, 3 * kLane)};
}
template <typename T, size_t N = 16 / sizeof(T)>
struct Indices128 {
__m128i raw;
};
template <class D, typename T = TFromD<D>, typename TI, size_t kN,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Indices128<T, kN> IndicesFromVec(D d, Vec128<TI, kN> vec) {
static_assert(sizeof(T) == sizeof(TI), "Index size must match lane");
(void)d;
return Indices128<T, kN>{vec.raw};
}
template <class D, typename T = TFromD<D>, typename TI, size_t kN,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Indices128<T, kN> IndicesFromVec(D d, Vec128<TI, kN> vec) {
static_assert(sizeof(T) == sizeof(TI), "Index size must match lane");
(void)d;
return Indices128<T, kN>{vec.raw};
}
template <class D, typename T = TFromD<D>, typename TI, size_t kN,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Indices128<T, kN> IndicesFromVec(D d, Vec128<TI, kN> vec) {
static_assert(sizeof(T) == sizeof(TI), "Index size must match lane");
(void)d;
return Indices128<T, kN>{vec.raw};
}
template <class D, typename T = TFromD<D>, typename TI, size_t kN,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Indices128<T, kN> IndicesFromVec(D d, Vec128<TI, kN> vec) {
static_assert(sizeof(T) == sizeof(TI), "Index size must match lane");
(void)d;
return Indices128<T, kN>{vec.raw};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename TI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Indices128<TFromD<D>, D::kPrivateLanes> SetTableIndices(
D d, const TI* idx) {
static_assert(sizeof(TFromD<D>) == sizeof(TI), "Index size must match lane");
const Rebind<TI, decltype(d)> di;
return IndicesFromVec(d, LoadU(di, idx));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> TableLookupLanes(Vec128<T, N> v, Indices128<T, N> idx) {
return TableLookupBytes(v, Vec128<T, N>{idx.raw});
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> TableLookupLanes(Vec128<T, N> v, Indices128<T, N> idx) {
typedef uint16_t GccU16RawVectType __attribute__((__vector_size__(16)));
return Vec128<T, N>{reinterpret_cast<typename detail::Raw128<T>::type>(
__builtin_shuffle(reinterpret_cast<GccU16RawVectType>(v.raw),
reinterpret_cast<GccU16RawVectType>(idx.raw)))};
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> TableLookupLanes(Vec128<T, N> v, Indices128<T, N> idx) {
typedef uint32_t GccU32RawVectType __attribute__((__vector_size__(16)));
return Vec128<T, N>{reinterpret_cast<typename detail::Raw128<T>::type>(
__builtin_shuffle(reinterpret_cast<GccU32RawVectType>(v.raw),
reinterpret_cast<GccU32RawVectType>(idx.raw)))};
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> TableLookupLanes(Vec128<T, 1> v,
Indices128<T, 1> ) {
return v;
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> TableLookupLanes(Vec128<T> v, Indices128<T> idx) {
const DFromV<decltype(v)> d;
Vec128<int64_t> vidx{idx.raw};
const RebindToSigned<decltype(d)> di;
const Vec128<int64_t> same = (vidx ^ Iota(di, 0)) - Set(di, 1);
const Mask128<T> mask_same = RebindMask(d, MaskFromVec(same));
return IfThenElse(mask_same, v, Shuffle01(v));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> TableLookupLanes(Vec128<double> v,
Indices128<double> idx) {
Vec128<int64_t> vidx{idx.raw};
const DFromV<decltype(v)> d;
const RebindToSigned<decltype(d)> di;
const Vec128<int64_t> same = (vidx ^ Iota(di, 0)) - Set(di, 1);
const Mask128<double> mask_same = RebindMask(d, MaskFromVec(same));
return IfThenElse(mask_same, v, Shuffle01(v));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReverseBlocks(D , VFromD<D> v) {
return v;
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> Reverse(D , Vec128<T, 1> v) {
return v;
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> Reverse(D , const Vec64<T> v) {
return Vec64<T>{Shuffle2301(Vec128<T>{v.raw}).raw};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Reverse(D , const Vec128<T> v) {
return Shuffle01(v);
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Reverse(D , const Vec128<T> v) {
return Shuffle0123(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse(D d, const VFromD<D> v) {
constexpr size_t kN = MaxLanes(d);
if (kN == 1) return v;
if (kN == 2) {
return VFromD<D>{_mm_shufflelo_epi16(v.raw,
(((
0
) << 6) | ((
1
) << 4) | ((
0
) << 2) | (
1
))
)};
}
if (kN == 4) {
return VFromD<D>{_mm_shufflelo_epi16(v.raw,
(((
0
) << 6) | ((
1
) << 4) | ((
2
) << 2) | (
3
))
)};
}
const VFromD<D> rev4{
_mm_shufflehi_epi16(_mm_shufflelo_epi16(v.raw,
(((
0
) << 6) | ((
1
) << 4) | ((
2
) << 2) | (
3
))
),
(((
0
) << 6) | ((
1
) << 4) | ((
2
) << 2) | (
3
))
)};
return VFromD<D>{_mm_shuffle_epi32(rev4.raw,
(((
1
) << 6) | ((
0
) << 4) | ((
3
) << 2) | (
2
))
)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse(D d, const VFromD<D> v) {
constexpr int kN = static_cast<int>(MaxLanes(d));
if (kN == 1) return v;
const RepartitionToWide<decltype(d)> d16;
return BitCast(d, Reverse(d16, RotateRight<8>(BitCast(d16, v))));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> Reverse2(D , Vec128<T, 1> v) {
return v;
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse2(D d, VFromD<D> v) {
constexpr size_t kN = MaxLanes(d);
__m128i shuf_result = _mm_shufflelo_epi16(v.raw,
(((
2
) << 6) | ((
3
) << 4) | ((
0
) << 2) | (
1
))
);
if (kN > 4) {
shuf_result = _mm_shufflehi_epi16(shuf_result,
(((
2
) << 6) | ((
3
) << 4) | ((
0
) << 2) | (
1
))
);
}
return VFromD<D>{shuf_result};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse2(D , VFromD<D> v) {
return Shuffle2301(v);
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse2(D , VFromD<D> v) {
return Shuffle01(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse4(D d, VFromD<D> v) {
constexpr size_t kN = MaxLanes(d);
if (kN <= 4) {
return VFromD<D>{_mm_shufflelo_epi16(v.raw,
(((
0
) << 6) | ((
1
) << 4) | ((
2
) << 2) | (
3
))
)};
}
return VFromD<D>{
_mm_shufflehi_epi16(_mm_shufflelo_epi16(v.raw,
(((
0
) << 6) | ((
1
) << 4) | ((
2
) << 2) | (
3
))
),
(((
0
) << 6) | ((
1
) << 4) | ((
2
) << 2) | (
3
))
)};
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse4(D , const VFromD<D> v) {
return Shuffle0123(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse4(D , VFromD<D> ) {
do { if (!(0)) { ::hwy::Abort("/home/mathieu/Perso/highway/hwy/ops/x86_128-inl.h", 4907, "Assert %s","0"); } } while (0);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse8(D d, const VFromD<D> v) {
const RepartitionToWide<decltype(d)> dw;
return Reverse2(d, BitCast(d, Shuffle0123(BitCast(dw, v))));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr,
hwy::EnableIf<((size_t{1} << sizeof(TFromD<D>)) & ((1 << 4) | (1 << 8))) != 0>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse8(D , VFromD<D> ) {
do { if (!(0)) { ::hwy::Abort("/home/mathieu/Perso/highway/hwy/ops/x86_128-inl.h", 4928, "Assert %s","0"); } } while (0);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> InterleaveLower(Vec128<uint8_t, N> a,
Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_unpacklo_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> InterleaveLower(Vec128<uint16_t, N> a,
Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_unpacklo_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> InterleaveLower(Vec128<uint32_t, N> a,
Vec128<uint32_t, N> b) {
return Vec128<uint32_t, N>{_mm_unpacklo_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> InterleaveLower(Vec128<uint64_t, N> a,
Vec128<uint64_t, N> b) {
return Vec128<uint64_t, N>{_mm_unpacklo_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> InterleaveLower(Vec128<int8_t, N> a,
Vec128<int8_t, N> b) {
return Vec128<int8_t, N>{_mm_unpacklo_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> InterleaveLower(Vec128<int16_t, N> a,
Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_unpacklo_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> InterleaveLower(Vec128<int32_t, N> a,
Vec128<int32_t, N> b) {
return Vec128<int32_t, N>{_mm_unpacklo_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> InterleaveLower(Vec128<int64_t, N> a,
Vec128<int64_t, N> b) {
return Vec128<int64_t, N>{_mm_unpacklo_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> InterleaveLower(Vec128<float, N> a,
Vec128<float, N> b) {
return Vec128<float, N>{_mm_unpacklo_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> InterleaveLower(Vec128<double, N> a,
Vec128<double, N> b) {
return Vec128<double, N>{_mm_unpacklo_pd(a.raw, b.raw)};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> InterleaveLower(D , VFromD<D> a, VFromD<D> b) {
return InterleaveLower(a, b);
}
namespace detail {
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t> InterleaveUpper(Vec128<uint8_t> a, Vec128<uint8_t> b) {
return Vec128<uint8_t>{_mm_unpackhi_epi8(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t> InterleaveUpper(Vec128<uint16_t> a,
Vec128<uint16_t> b) {
return Vec128<uint16_t>{_mm_unpackhi_epi16(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> InterleaveUpper(Vec128<uint32_t> a,
Vec128<uint32_t> b) {
return Vec128<uint32_t>{_mm_unpackhi_epi32(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t> InterleaveUpper(Vec128<uint64_t> a,
Vec128<uint64_t> b) {
return Vec128<uint64_t>{_mm_unpackhi_epi64(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t> InterleaveUpper(Vec128<int8_t> a, Vec128<int8_t> b) {
return Vec128<int8_t>{_mm_unpackhi_epi8(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t> InterleaveUpper(Vec128<int16_t> a, Vec128<int16_t> b) {
return Vec128<int16_t>{_mm_unpackhi_epi16(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> InterleaveUpper(Vec128<int32_t> a, Vec128<int32_t> b) {
return Vec128<int32_t>{_mm_unpackhi_epi32(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t> InterleaveUpper(Vec128<int64_t> a, Vec128<int64_t> b) {
return Vec128<int64_t>{_mm_unpackhi_epi64(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> InterleaveUpper(Vec128<float> a, Vec128<float> b) {
return Vec128<float>{_mm_unpackhi_ps(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> InterleaveUpper(Vec128<double> a, Vec128<double> b) {
return Vec128<double>{_mm_unpackhi_pd(a.raw, b.raw)};
}
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> InterleaveUpper(D , Vec128<T> a, Vec128<T> b) {
return detail::InterleaveUpper(a, b);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> InterleaveUpper(D d, VFromD<D> a, VFromD<D> b) {
const Half<decltype(d)> d2;
return InterleaveLower(d, VFromD<D>{UpperHalf(d2, a).raw},
VFromD<D>{UpperHalf(d2, b).raw});
}
template <int kLane, class T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Broadcast(const Vec128<T, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
const DFromV<decltype(v)> d;
const Full128<T> d_full;
const Vec128<T> v_full{v.raw};
const auto v_interleaved = (kLane < 8)
? InterleaveLower(d_full, v_full, v_full)
: InterleaveUpper(d_full, v_full, v_full);
return ResizeBitCast(
d, Broadcast<kLane & 7>(BitCast(Full128<uint16_t>(), v_interleaved)));
}
template <class V, class DW = RepartitionToWide<DFromV<V>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DW> ZipLower(V a, V b) {
return BitCast(DW(), InterleaveLower(a, b));
}
template <class V, class D = DFromV<V>, class DW = RepartitionToWide<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DW> ZipLower(DW dw, V a, V b) {
return BitCast(dw, InterleaveLower(D(), a, b));
}
template <class V, class D = DFromV<V>, class DW = RepartitionToWide<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DW> ZipUpper(DW dw, V a, V b) {
return BitCast(dw, InterleaveUpper(D(), a, b));
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Per4LaneBlkShufDupSet4xU32(D d, const uint32_t x3,
const uint32_t x2,
const uint32_t x1,
const uint32_t x0) {
return ResizeBitCast(
d, Vec128<uint32_t>{_mm_set_epi32(
static_cast<int32_t>(x3), static_cast<int32_t>(x2),
static_cast<int32_t>(x1), static_cast<int32_t>(x0))});
}
template <size_t kIdx3210, class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> ,
hwy::SizeTag<2> ,
hwy::SizeTag<8> , V v) {
return V{_mm_shufflelo_epi16(v.raw, static_cast<int>(kIdx3210 & 0xFF))};
}
template <size_t kIdx3210, class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> ,
hwy::SizeTag<2> ,
hwy::SizeTag<16> , V v) {
constexpr int kShuffle = static_cast<int>(kIdx3210 & 0xFF);
return V{_mm_shufflehi_epi16(_mm_shufflelo_epi16(v.raw, kShuffle), kShuffle)};
}
template <size_t kIdx3210, size_t kVectSize, class V,
hwy::EnableIf<(kVectSize == 4 || kVectSize == 8)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> idx_3210_tag,
hwy::SizeTag<1> ,
hwy::SizeTag<kVectSize> ,
V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const Rebind<uint16_t, decltype(d)> du16;
const RebindToSigned<decltype(du16)> di16;
const auto vu16 = PromoteTo(du16, BitCast(du, v));
const auto shuf16_result = Per4LaneBlockShuffle(
idx_3210_tag, hwy::SizeTag<2>(), hwy::SizeTag<kVectSize * 2>(), vu16);
return BitCast(d, DemoteTo(du, BitCast(di16, shuf16_result)));
}
template <size_t kIdx3210, size_t kVectSize, class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> idx_3210_tag,
hwy::SizeTag<1> ,
hwy::SizeTag<16> , V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const Repartition<uint16_t, decltype(d)> du16;
const RebindToSigned<decltype(du16)> di16;
const auto zero = Zero(d);
const auto v_lo16 = BitCast(du16, InterleaveLower(d, v, zero));
const auto v_hi16 = BitCast(du16, InterleaveUpper(d, v, zero));
const auto lo_shuf_result = Per4LaneBlockShuffle(
idx_3210_tag, hwy::SizeTag<2>(), hwy::SizeTag<16>(), v_lo16);
const auto hi_shuf_result = Per4LaneBlockShuffle(
idx_3210_tag, hwy::SizeTag<2>(), hwy::SizeTag<16>(), v_hi16);
return BitCast(d, OrderedDemote2To(du, BitCast(di16, lo_shuf_result),
BitCast(di16, hi_shuf_result)));
}
template <size_t kIdx3210, class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >()>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> ,
hwy::SizeTag<4> ,
hwy::SizeTag<16> , V v) {
return V{_mm_shuffle_epi32(v.raw, static_cast<int>(kIdx3210 & 0xFF))};
}
template <size_t kIdx3210, class V, hwy::EnableIf<hwy::IsFloat<TFromV<V> >()>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> ,
hwy::SizeTag<4> ,
hwy::SizeTag<16> , V v) {
return V{_mm_shuffle_ps(v.raw, v.raw, static_cast<int>(kIdx3210 & 0xFF))};
}
}
namespace detail {
template <class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 8>* = nullptr>
inline __attribute__((always_inline)) V SlideUpLanes(V v, size_t amt) {
const DFromV<decltype(v)> d;
const Full64<uint64_t> du64;
const auto vu64 = ResizeBitCast(du64, v);
return ResizeBitCast(
d, ShiftLeftSame(vu64, static_cast<int>(amt * sizeof(TFromV<V>) * 8)));
}
template <class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) == 16>* = nullptr>
inline __attribute__((always_inline)) V SlideUpLanes(V v, size_t amt) {
const DFromV<decltype(v)> d;
const Repartition<int32_t, decltype(d)> di32;
const Repartition<uint64_t, decltype(d)> du64;
constexpr size_t kNumOfLanesPerU64 = 8 / sizeof(TFromV<V>);
const auto vu64 = BitCast(du64, v);
const auto v_hi = IfVecThenElse(
BitCast(du64, Set(di32, -static_cast<int32_t>(amt >= kNumOfLanesPerU64))),
BitCast(du64, ShiftLeftBytes<8>(du64, vu64)), vu64);
const auto v_lo = ShiftLeftBytes<8>(du64, v_hi);
const int shl_amt = static_cast<int>((amt * sizeof(TFromV<V>) * 8) & 63);
return BitCast(
d, Or(ShiftLeftSame(v_hi, shl_amt), ShiftRightSame(v_lo, 64 - shl_amt)));
}
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpLanes(D , VFromD<D> v, size_t ) {
return v;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftLeftLanes<1>(d, v);
}
}
return detail::SlideUpLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftLeftLanes<1>(d, v);
case 2:
return ShiftLeftLanes<2>(d, v);
case 3:
return ShiftLeftLanes<3>(d, v);
}
}
return detail::SlideUpLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftLeftLanes<1>(d, v);
case 2:
return ShiftLeftLanes<2>(d, v);
case 3:
return ShiftLeftLanes<3>(d, v);
case 4:
return ShiftLeftLanes<4>(d, v);
case 5:
return ShiftLeftLanes<5>(d, v);
case 6:
return ShiftLeftLanes<6>(d, v);
case 7:
return ShiftLeftLanes<7>(d, v);
}
}
return detail::SlideUpLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 16)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftLeftLanes<1>(d, v);
case 2:
return ShiftLeftLanes<2>(d, v);
case 3:
return ShiftLeftLanes<3>(d, v);
case 4:
return ShiftLeftLanes<4>(d, v);
case 5:
return ShiftLeftLanes<5>(d, v);
case 6:
return ShiftLeftLanes<6>(d, v);
case 7:
return ShiftLeftLanes<7>(d, v);
case 8:
return ShiftLeftLanes<8>(d, v);
case 9:
return ShiftLeftLanes<9>(d, v);
case 10:
return ShiftLeftLanes<10>(d, v);
case 11:
return ShiftLeftLanes<11>(d, v);
case 12:
return ShiftLeftLanes<12>(d, v);
case 13:
return ShiftLeftLanes<13>(d, v);
case 14:
return ShiftLeftLanes<14>(d, v);
case 15:
return ShiftLeftLanes<15>(d, v);
}
}
return detail::SlideUpLanes(v, amt);
}
namespace detail {
template <class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 8>* = nullptr>
inline __attribute__((always_inline)) V SlideDownLanes(V v, size_t amt) {
const DFromV<decltype(v)> d;
const Repartition<UnsignedFromSize<d.MaxBytes()>, decltype(d)> dv;
return BitCast(d,
ShiftRightSame(BitCast(dv, v),
static_cast<int>(amt * sizeof(TFromV<V>) * 8)));
}
template <class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) == 16>* = nullptr>
inline __attribute__((always_inline)) V SlideDownLanes(V v, size_t amt) {
const DFromV<decltype(v)> d;
const Repartition<int32_t, decltype(d)> di32;
const Repartition<uint64_t, decltype(d)> du64;
constexpr size_t kNumOfLanesPerU64 = 8 / sizeof(TFromV<V>);
const auto vu64 = BitCast(du64, v);
const auto v_lo = IfVecThenElse(
BitCast(du64, Set(di32, -static_cast<int32_t>(amt >= kNumOfLanesPerU64))),
BitCast(du64, ShiftRightBytes<8>(du64, vu64)), vu64);
const auto v_hi = ShiftRightBytes<8>(du64, v_lo);
const int shr_amt = static_cast<int>((amt * sizeof(TFromV<V>) * 8) & 63);
return BitCast(
d, Or(ShiftRightSame(v_lo, shr_amt), ShiftLeftSame(v_hi, 64 - shr_amt)));
}
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownLanes(D , VFromD<D> v, size_t ) {
return v;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftRightLanes<1>(d, v);
}
}
return detail::SlideDownLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftRightLanes<1>(d, v);
case 2:
return ShiftRightLanes<2>(d, v);
case 3:
return ShiftRightLanes<3>(d, v);
}
}
return detail::SlideDownLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftRightLanes<1>(d, v);
case 2:
return ShiftRightLanes<2>(d, v);
case 3:
return ShiftRightLanes<3>(d, v);
case 4:
return ShiftRightLanes<4>(d, v);
case 5:
return ShiftRightLanes<5>(d, v);
case 6:
return ShiftRightLanes<6>(d, v);
case 7:
return ShiftRightLanes<7>(d, v);
}
}
return detail::SlideDownLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 16)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftRightLanes<1>(d, v);
case 2:
return ShiftRightLanes<2>(d, v);
case 3:
return ShiftRightLanes<3>(d, v);
case 4:
return ShiftRightLanes<4>(d, v);
case 5:
return ShiftRightLanes<5>(d, v);
case 6:
return ShiftRightLanes<6>(d, v);
case 7:
return ShiftRightLanes<7>(d, v);
case 8:
return ShiftRightLanes<8>(d, v);
case 9:
return ShiftRightLanes<9>(d, v);
case 10:
return ShiftRightLanes<10>(d, v);
case 11:
return ShiftRightLanes<11>(d, v);
case 12:
return ShiftRightLanes<12>(d, v);
case 13:
return ShiftRightLanes<13>(d, v);
case 14:
return ShiftRightLanes<14>(d, v);
case 15:
return ShiftRightLanes<15>(d, v);
}
}
return detail::SlideDownLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, class VH = VFromD<Half<D>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Combine(D d, VH hi_half, VH lo_half) {
const Half<decltype(d)> dh;
const RebindToUnsigned<decltype(dh)> duh;
using VU = Vec128<UnsignedFromSize<dh.MaxBytes()>, 2>;
const VU lo{BitCast(duh, lo_half).raw};
const VU hi{BitCast(duh, hi_half).raw};
return BitCast(d, InterleaveLower(lo, hi));
}
namespace detail {
template <class D, typename T = TFromD<D>>
inline __attribute__((always_inline)) Vec128<T> ZeroExtendVector(hwy::NonFloatTag , D ,
Vec64<T> lo) {
return Vec128<T>{_mm_move_epi64(lo.raw)};
}
template <class D, typename T = TFromD<D>>
inline __attribute__((always_inline)) Vec128<T> ZeroExtendVector(hwy::FloatTag , D d, Vec64<T> lo) {
const RebindToUnsigned<decltype(d)> du;
return BitCast(d, ZeroExtendVector(du, BitCast(Half<decltype(du)>(), lo)));
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ZeroExtendVector(D d, Vec64<T> lo) {
return detail::ZeroExtendVector(hwy::IsFloatTag<T>(), d, lo);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ZeroExtendVector(D d, VFromD<Half<D>> lo) {
const Half<D> dh;
return IfThenElseZero(FirstN(d, MaxLanes(dh)), VFromD<D>{lo.raw});
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatLowerLower(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<uint64_t, decltype(d)> d64;
return BitCast(d, InterleaveLower(BitCast(d64, lo), BitCast(d64, hi)));
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatUpperUpper(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<uint64_t, decltype(d)> d64;
return BitCast(d, InterleaveUpper(d64, BitCast(d64, lo), BitCast(d64, hi)));
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatLowerUpper(D d, Vec128<T> hi, Vec128<T> lo) {
return CombineShiftRightBytes<8>(d, hi, lo);
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatUpperLower(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<double, decltype(d)> dd;
return BitCast(
d, Vec128<double>{_mm_shuffle_pd(BitCast(dd, lo).raw, BitCast(dd, hi).raw,
(((
1
) << 1) | (
0
))
)});
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> ConcatUpperLower(D d, Vec128<float> hi,
Vec128<float> lo) {
(void)d;
return Vec128<float>{_mm_shuffle_ps(lo.raw, hi.raw,
(((
3
) << 6) | ((
2
) << 4) | ((
1
) << 2) | (
0
))
)};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> ConcatUpperLower(D , Vec128<double> hi,
Vec128<double> lo) {
return Vec128<double>{_mm_shuffle_pd(lo.raw, hi.raw,
(((
1
) << 1) | (
0
))
)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatLowerLower(D d, VFromD<D> hi, VFromD<D> lo) {
const Half<decltype(d)> d2;
return Combine(d, LowerHalf(d2, hi), LowerHalf(d2, lo));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatUpperUpper(D d, VFromD<D> hi, VFromD<D> lo) {
const Half<decltype(d)> d2;
return Combine(d, UpperHalf(d2, hi), UpperHalf(d2, lo));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatLowerUpper(D d, const VFromD<D> hi,
const VFromD<D> lo) {
const Half<decltype(d)> d2;
return Combine(d, LowerHalf(d2, hi), UpperHalf(d2, lo));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatUpperLower(D d, VFromD<D> hi, VFromD<D> lo) {
const Half<decltype(d)> d2;
return Combine(d, UpperHalf(d2, hi), LowerHalf(d2, lo));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatOdd(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<uint16_t, decltype(d)> dw;
const Vec128<uint16_t> uH = ShiftRight<8>(BitCast(dw, hi));
const Vec128<uint16_t> uL = ShiftRight<8>(BitCast(dw, lo));
return Vec128<T>{_mm_packus_epi16(uL.raw, uH.raw)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ConcatOdd(D d, Vec64<T> hi, Vec64<T> lo) {
const Repartition<uint16_t, decltype(d)> dw;
const Vec64<uint16_t> uH = ShiftRight<8>(BitCast(dw, hi));
const Vec64<uint16_t> uL = ShiftRight<8>(BitCast(dw, lo));
return Vec64<T>{_mm_shuffle_epi32(_mm_packus_epi16(uL.raw, uH.raw),
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<T> ConcatOdd(D d, Vec32<T> hi, Vec32<T> lo) {
const Repartition<uint16_t, decltype(d)> dw;
const Twice<decltype(dw)> dw_2;
const Vec32<uint16_t> uH = ShiftRight<8>(BitCast(dw, hi));
const Vec32<uint16_t> uL = ShiftRight<8>(BitCast(dw, lo));
const Vec64<uint16_t> uHL = Combine(dw_2, uH, uL);
return Vec32<T>{_mm_packus_epi16(uHL.raw, uHL.raw)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatOdd(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<int32_t, decltype(d)> dw;
const Vec128<int32_t> uH = ShiftRight<16>(BitCast(dw, hi));
const Vec128<int32_t> uL = ShiftRight<16>(BitCast(dw, lo));
return Vec128<T>{_mm_packs_epi32(uL.raw, uH.raw)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ConcatOdd(D d, Vec64<T> hi, Vec64<T> lo) {
const Repartition<int32_t, decltype(d)> dw;
const Vec64<int32_t> uH = ShiftRight<16>(BitCast(dw, hi));
const Vec64<int32_t> uL = ShiftRight<16>(BitCast(dw, lo));
return Vec64<T>{_mm_shuffle_epi32(_mm_packs_epi32(uL.raw, uH.raw),
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatOdd(D d, Vec128<T> hi, Vec128<T> lo) {
const RebindToFloat<decltype(d)> df;
return BitCast(
d, Vec128<float>{_mm_shuffle_ps(BitCast(df, lo).raw, BitCast(df, hi).raw,
(((
3
) << 6) | ((
1
) << 4) | ((
3
) << 2) | (
1
))
)});
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> ConcatOdd(D , Vec128<float> hi, Vec128<float> lo) {
return Vec128<float>{_mm_shuffle_ps(lo.raw, hi.raw,
(((
3
) << 6) | ((
1
) << 4) | ((
3
) << 2) | (
1
))
)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 2> ConcatOdd(D d, Vec128<T, 2> hi, Vec128<T, 2> lo) {
return InterleaveUpper(d, lo, hi);
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatEven(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<uint16_t, decltype(d)> dw;
const Vec128<uint16_t> mask = Set(dw, 0x00FF);
const Vec128<uint16_t> uH = And(BitCast(dw, hi), mask);
const Vec128<uint16_t> uL = And(BitCast(dw, lo), mask);
return Vec128<T>{_mm_packus_epi16(uL.raw, uH.raw)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ConcatEven(D d, Vec64<T> hi, Vec64<T> lo) {
const Repartition<uint16_t, decltype(d)> dw;
const Vec64<uint16_t> mask = Set(dw, 0x00FF);
const Vec64<uint16_t> uH = And(BitCast(dw, hi), mask);
const Vec64<uint16_t> uL = And(BitCast(dw, lo), mask);
return Vec64<T>{_mm_shuffle_epi32(_mm_packus_epi16(uL.raw, uH.raw),
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<T> ConcatEven(D d, Vec32<T> hi, Vec32<T> lo) {
const Repartition<uint16_t, decltype(d)> dw;
const Twice<decltype(dw)> dw_2;
const Vec32<uint16_t> mask = Set(dw, 0x00FF);
const Vec32<uint16_t> uH = And(BitCast(dw, hi), mask);
const Vec32<uint16_t> uL = And(BitCast(dw, lo), mask);
const Vec64<uint16_t> uHL = Combine(dw_2, uH, uL);
return Vec32<T>{_mm_packus_epi16(uHL.raw, uHL.raw)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatEven(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<uint32_t, decltype(d)> dw;
return ConcatOdd(d, BitCast(d, ShiftLeft<16>(BitCast(dw, hi))),
BitCast(d, ShiftLeft<16>(BitCast(dw, lo))));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ConcatEven(D d, Vec64<T> hi, Vec64<T> lo) {
const Repartition<uint32_t, decltype(d)> dw;
return ConcatOdd(d, BitCast(d, ShiftLeft<16>(BitCast(dw, hi))),
BitCast(d, ShiftLeft<16>(BitCast(dw, lo))));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatEven(D d, Vec128<T> hi, Vec128<T> lo) {
const RebindToFloat<decltype(d)> df;
return BitCast(
d, Vec128<float>{_mm_shuffle_ps(BitCast(df, lo).raw, BitCast(df, hi).raw,
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)});
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> ConcatEven(D , Vec128<float> hi,
Vec128<float> lo) {
return Vec128<float>{_mm_shuffle_ps(lo.raw, hi.raw,
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)};
}
template <typename D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 2> ConcatEven(D d, Vec128<T, 2> hi, Vec128<T, 2> lo) {
return InterleaveLower(d, lo, hi);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> DupEven(const Vec128<T, 1> v) {
return v;
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 2> DupEven(const Vec128<T, 2> v) {
return InterleaveLower(DFromV<decltype(v)>(), v, v);
}
template <typename V, hwy::EnableIf<sizeof(TFromV<V>) == (1)>* = nullptr, hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) > 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V DupEven(V v) {
const DFromV<decltype(v)> d;
const Repartition<uint16_t, decltype(d)> du16;
return IfVecThenElse(BitCast(d, Set(du16, uint16_t{0xFF00})),
BitCast(d, ShiftLeft<8>(BitCast(du16, v))), v);
}
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> DupEven(const Vec64<T> v) {
return Vec64<T>{_mm_shufflelo_epi16(v.raw,
(((
2
) << 6) | ((
2
) << 4) | ((
0
) << 2) | (
0
))
)};
}
template <typename V, hwy::EnableIf<sizeof(TFromV<V>) == (2)>* = nullptr, hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V DupEven(V v) {
return V{
_mm_shufflehi_epi16(_mm_shufflelo_epi16(v.raw,
(((
2
) << 6) | ((
2
) << 4) | ((
0
) << 2) | (
0
))
),
(((
2
) << 6) | ((
2
) << 4) | ((
0
) << 2) | (
0
))
)};
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> DupEven(Vec128<T> v) {
return Vec128<T>{_mm_shuffle_epi32(v.raw,
(((
2
) << 6) | ((
2
) << 4) | ((
0
) << 2) | (
0
))
)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> DupEven(Vec128<float> v) {
return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw,
(((
2
) << 6) | ((
2
) << 4) | ((
0
) << 2) | (
0
))
)};
}
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> DupOdd(Vec128<T, 1> v) {
return v;
}
template <typename V, hwy::EnableIf<sizeof(TFromV<V>) == (1)>* = nullptr, hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V DupOdd(V v) {
const DFromV<decltype(v)> d;
const Repartition<uint16_t, decltype(d)> du16;
return IfVecThenElse(BitCast(d, Set(du16, uint16_t{0x00FF})),
BitCast(d, ShiftRight<8>(BitCast(du16, v))), v);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr, hwy::EnableIf<(N <= 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> DupOdd(Vec128<T, N> v) {
return Vec128<T, N>{_mm_shufflelo_epi16(v.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
}
template <typename V, hwy::EnableIf<sizeof(TFromV<V>) == (2)>* = nullptr, hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V DupOdd(V v) {
return V{
_mm_shufflehi_epi16(_mm_shufflelo_epi16(v.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
),
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> DupOdd(Vec128<T, N> v) {
return Vec128<T, N>{_mm_shuffle_epi32(v.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> DupOdd(Vec128<float, N> v) {
return Vec128<float, N>{
_mm_shuffle_ps(v.raw, v.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> DupOdd(const Vec128<T, N> v) {
return InterleaveUpper(DFromV<decltype(v)>(), v, v);
}
template <typename T, size_t N, hwy::EnableIf<N * sizeof(T) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> TwoTablesLookupLanes(Vec128<T, N> a, Vec128<T, N> b,
Indices128<T, N> idx) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
const Indices128<T, N * 2> idx2{idx.raw};
return LowerHalf(d, TableLookupLanes(Combine(dt, b, a), idx2));
}
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> TwoTablesLookupLanes(Vec128<T> a, Vec128<T> b,
Indices128<T> idx) {
const DFromV<decltype(a)> d;
const Vec128<T> idx_vec{idx.raw};
const RebindToSigned<decltype(d)> di;
const auto sel_hi_mask =
RebindMask(d, BitCast(di, idx_vec) > Set(di, int8_t{15}));
const auto lo_lookup_result = TableLookupBytes(a, idx_vec);
const auto hi_lookup_result = TableLookupBytes(b, idx_vec);
return IfThenElse(sel_hi_mask, hi_lookup_result, lo_lookup_result);
}
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> TwoTablesLookupLanes(Vec128<T> a, Vec128<T> b,
Indices128<T> idx) {
const DFromV<decltype(a)> d;
const RebindToSigned<decltype(d)> di;
const Vec128<T> idx_vec{idx.raw};
const auto sel_hi_mask =
RebindMask(d, BitCast(di, idx_vec) > Set(di, int16_t{7}));
const auto lo_lookup_result = TableLookupLanes(a, idx);
const auto hi_lookup_result = TableLookupLanes(b, idx);
return IfThenElse(sel_hi_mask, hi_lookup_result, lo_lookup_result);
}
template <typename T, hwy::EnableIf<IsSame<T, uint32_t>() || IsSame<T, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> TwoTablesLookupLanes(Vec128<T> a, Vec128<T> b,
Indices128<T> idx) {
const DFromV<decltype(a)> d;
const Vec128<T> idx_vec{idx.raw};
const RebindToSigned<decltype(d)> d_sel;
const auto sel_hi_mask = BitCast(d_sel, idx_vec) > Set(d_sel, int32_t{3});
const auto lo_lookup_result = BitCast(d_sel, TableLookupLanes(a, idx));
const auto hi_lookup_result = BitCast(d_sel, TableLookupLanes(b, idx));
return BitCast(d,
IfThenElse(sel_hi_mask, hi_lookup_result, lo_lookup_result));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> TwoTablesLookupLanes(Vec128<float> a, Vec128<float> b,
Indices128<float> idx) {
const DFromV<decltype(a)> d;
const RebindToSigned<decltype(d)> di;
const auto sel_hi_mask =
RebindMask(d, Vec128<int32_t>{idx.raw} > Set(di, int32_t{3}));
const auto lo_lookup_result = TableLookupLanes(a, idx);
const auto hi_lookup_result = TableLookupLanes(b, idx);
return IfThenElse(sel_hi_mask, hi_lookup_result, lo_lookup_result);
}
template <typename T, hwy::EnableIf<IsSame<T, uint64_t>() || IsSame<T, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> TwoTablesLookupLanes(Vec128<T> a, Vec128<T> b,
Indices128<T> idx) {
const DFromV<decltype(a)> d;
const Vec128<T> idx_vec{idx.raw};
const Indices128<T> idx_mod{And(idx_vec, Set(d, T{1})).raw};
const Repartition<int32_t, decltype(d)> di32;
const RebindToSigned<decltype(d)> d_sel;
const auto sel_hi_mask = MaskFromVec(
BitCast(d_sel, VecFromMask(di32, DupEven(BitCast(di32, idx_vec)) >
Set(di32, int32_t{1}))));
const auto lo_lookup_result = BitCast(d_sel, TableLookupLanes(a, idx_mod));
const auto hi_lookup_result = BitCast(d_sel, TableLookupLanes(b, idx_mod));
return BitCast(d,
IfThenElse(sel_hi_mask, hi_lookup_result, lo_lookup_result));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> TwoTablesLookupLanes(Vec128<double> a, Vec128<double> b,
Indices128<double> idx) {
const DFromV<decltype(a)> d;
const RebindToSigned<decltype(d)> di;
const Vec128<int64_t> idx_vec{idx.raw};
const Indices128<double> idx_mod{And(idx_vec, Set(di, int64_t{1})).raw};
const Repartition<int32_t, decltype(d)> di32;
const auto sel_hi_mask =
MaskFromVec(BitCast(d, VecFromMask(di32, DupEven(BitCast(di32, idx_vec)) >
Set(di32, int32_t{1}))));
const auto lo_lookup_result = TableLookupLanes(a, idx_mod);
const auto hi_lookup_result = TableLookupLanes(b, idx_mod);
return IfThenElse(sel_hi_mask, hi_lookup_result, lo_lookup_result);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> OddEven(const Vec128<T, N> a, const Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const Repartition<uint8_t, decltype(d)> d8;
alignas(16) static constexpr uint8_t mask[16] = {
0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0};
return IfThenElse(MaskFromVec(BitCast(d, Load(d8, mask))), b, a);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> OddEven(const Vec128<T, N> a, const Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const Repartition<uint8_t, decltype(d)> d8;
alignas(16) static constexpr uint8_t mask[16] = {
0xFF, 0xFF, 0, 0, 0xFF, 0xFF, 0, 0, 0xFF, 0xFF, 0, 0, 0xFF, 0xFF, 0, 0};
return IfThenElse(MaskFromVec(BitCast(d, Load(d8, mask))), b, a);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> OddEven(const Vec128<T, N> a, const Vec128<T, N> b) {
const __m128i odd = _mm_shuffle_epi32(a.raw,
(((
3
) << 6) | ((
1
) << 4) | ((
3
) << 2) | (
1
))
);
const __m128i even = _mm_shuffle_epi32(b.raw,
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
);
return Vec128<T, N>{_mm_unpacklo_epi32(even, odd)};
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> OddEven(const Vec128<T, N> a, const Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const RebindToFloat<decltype(d)> dd;
return BitCast(
d, Vec128<double, N>{_mm_shuffle_pd(
BitCast(dd, b).raw, BitCast(dd, a).raw,
(((
1
) << 1) | (
0
))
)});
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> OddEven(Vec128<float, N> a, Vec128<float, N> b) {
const __m128 odd = _mm_shuffle_ps(a.raw, a.raw,
(((
3
) << 6) | ((
1
) << 4) | ((
3
) << 2) | (
1
))
);
const __m128 even = _mm_shuffle_ps(b.raw, b.raw,
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
);
return Vec128<float, N>{_mm_unpacklo_ps(even, odd)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> OddEvenBlocks(Vec128<T, N> , Vec128<T, N> even) {
return even;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> SwapAdjacentBlocks(Vec128<T, N> v) {
return v;
}
namespace detail {
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<MakeUnsigned<T>> Pow2(const Vec128<T> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const RepartitionToWide<decltype(d)> dw;
const Rebind<float, decltype(dw)> df;
const auto zero = Zero(d);
const auto exp = ShiftLeft<23 - 16>(v);
const auto upper = exp + Set(d, 0x3F80);
const auto f0 = ZipLower(dw, zero, upper);
const auto f1 = ZipUpper(dw, zero, upper);
const VFromD<decltype(dw)> bits0{_mm_cvtps_epi32(BitCast(df, f0).raw)};
const VFromD<decltype(dw)> bits1{_mm_cvtps_epi32(BitCast(df, f1).raw)};
return ConcatEven(du, BitCast(du, bits1), BitCast(du, bits0));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr, hwy::EnableIf<(N <= 4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<MakeUnsigned<T>, N> Pow2(const Vec128<T, N> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const Twice<decltype(du)> dt_u;
const RepartitionToWide<decltype(dt_u)> dt_w;
const RebindToFloat<decltype(dt_w)> dt_f;
const auto exp = ShiftLeft<23 - 16>(v);
const auto upper = exp + Set(d, 0x3F80);
const auto f0 = ZipLower(dt_w, Zero(dt_u), ResizeBitCast(dt_u, upper));
const VFromD<decltype(dt_w)> bits0{_mm_cvtps_epi32(BitCast(dt_f, f0).raw)};
const RebindToSigned<decltype(dt_w)> dt_i32;
const auto bits0_i32 = ShiftRight<16>(BitCast(dt_i32, ShiftLeft<16>(bits0)));
return VFromD<decltype(du)>{_mm_packs_epi32(bits0_i32.raw, bits0_i32.raw)};
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<MakeUnsigned<T>, N> Pow2(const Vec128<T, N> v) {
const DFromV<decltype(v)> d;
const auto exp = ShiftLeft<23>(v);
const auto f = exp + Set(d, 0x3F800000);
return Vec128<MakeUnsigned<T>, N>{_mm_cvtps_epi32(_mm_castsi128_ps(f.raw))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> Shl(hwy::UnsignedTag , Vec128<uint16_t, N> v,
Vec128<uint16_t, N> bits) {
return v * Pow2(bits);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec16<uint16_t> Shl(hwy::UnsignedTag , Vec16<uint16_t> v,
Vec16<uint16_t> bits) {
const auto bits16 = And(bits, Vec16<uint16_t>{_mm_set_epi64x(0, 0xFFFF)});
return Vec16<uint16_t>{_mm_sll_epi16(v.raw, bits16.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> Shl(hwy::UnsignedTag tag, Vec128<uint8_t, N> v,
Vec128<uint8_t, N> bits) {
const DFromV<decltype(v)> d;
const Repartition<uint16_t, decltype(d)> dw;
using VW = VFromD<decltype(dw)>;
const VW even_mask = Set(dw, 0x00FF);
const VW odd_mask = Set(dw, 0xFF00);
const VW vw = BitCast(dw, v);
const VW bits16 = BitCast(dw, bits);
const VW evens = Shl(tag, vw, And(bits16, even_mask));
const VW odds = Shl(tag, And(vw, odd_mask), ShiftRight<8>(bits16));
return OddEven(BitCast(d, odds), BitCast(d, evens));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, 1> Shl(hwy::UnsignedTag , Vec128<uint8_t, 1> v,
Vec128<uint8_t, 1> bits) {
const Vec16<uint16_t> bits8 =
And(Vec16<uint16_t>{bits.raw}, Vec16<uint16_t>{_mm_set_epi64x(0, 0xFF)});
return Vec128<uint8_t, 1>{_mm_sll_epi16(v.raw, bits8.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> Shl(hwy::UnsignedTag , Vec128<uint32_t, N> v,
Vec128<uint32_t, N> bits) {
return v * Pow2(bits);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<uint32_t> Shl(hwy::UnsignedTag , Vec32<uint32_t> v,
const Vec32<uint32_t> bits) {
const auto bits32 =
Combine(Full64<uint32_t>(), Zero(Full32<uint32_t>()), bits);
return Vec32<uint32_t>{_mm_sll_epi32(v.raw, bits32.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t> Shl(hwy::UnsignedTag , Vec128<uint64_t> v,
Vec128<uint64_t> bits) {
const DFromV<decltype(v)> d;
const Vec128<uint64_t> out0{_mm_sll_epi64(v.raw, bits.raw)};
const __m128i bits1 = _mm_unpackhi_epi64(bits.raw, bits.raw);
const Vec128<uint64_t> out1{_mm_sll_epi64(v.raw, bits1)};
return ConcatUpperLower(d, out1, out0);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<uint64_t> Shl(hwy::UnsignedTag , Vec64<uint64_t> v,
Vec64<uint64_t> bits) {
return Vec64<uint64_t>{_mm_sll_epi64(v.raw, bits.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Shl(hwy::SignedTag , Vec128<T, N> v,
Vec128<T, N> bits) {
const DFromV<decltype(v)> di;
const RebindToUnsigned<decltype(di)> du;
return BitCast(di,
Shl(hwy::UnsignedTag(), BitCast(du, v), BitCast(du, bits)));
}
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator<<(Vec128<T, N> v, Vec128<T, N> bits) {
return detail::Shl(hwy::TypeTag<T>(), v, bits);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> operator>>(Vec128<uint16_t, N> in,
const Vec128<uint16_t, N> bits) {
const DFromV<decltype(in)> d;
const auto out = MulHigh(in, detail::Pow2(Set(d, 16) - bits));
return IfThenElse(bits == Zero(d), in, out);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec16<uint16_t> operator>>(const Vec16<uint16_t> in,
const Vec16<uint16_t> bits) {
const auto bits16 = And(bits, Vec16<uint16_t>{_mm_set_epi64x(0, 0xFFFF)});
return Vec16<uint16_t>{_mm_srl_epi16(in.raw, bits16.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> operator>>(Vec128<uint8_t, N> in,
const Vec128<uint8_t, N> bits) {
const DFromV<decltype(in)> d;
const Repartition<uint16_t, decltype(d)> dw;
using VW = VFromD<decltype(dw)>;
const VW mask = Set(dw, 0x00FF);
const VW vw = BitCast(dw, in);
const VW bits16 = BitCast(dw, bits);
const VW evens = And(vw, mask) >> And(bits16, mask);
const VW odds = vw >> ShiftRight<8>(bits16);
return OddEven(BitCast(d, odds), BitCast(d, evens));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, 1> operator>>(const Vec128<uint8_t, 1> in,
const Vec128<uint8_t, 1> bits) {
const Vec16<uint16_t> mask{_mm_set_epi64x(0, 0xFF)};
const Vec16<uint16_t> in8 = And(Vec16<uint16_t>{in.raw}, mask);
const Vec16<uint16_t> bits8 = And(Vec16<uint16_t>{bits.raw}, mask);
return Vec128<uint8_t, 1>{_mm_srl_epi16(in8.raw, bits8.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> operator>>(const Vec128<uint32_t, N> in,
const Vec128<uint32_t, N> bits) {
const DFromV<decltype(in)> d32;
const Vec128<uint32_t, N> in31{_mm_shuffle_epi32(in.raw, 0x31)};
const auto mul = detail::Pow2(Set(d32, 32) - bits);
const auto out20 = ShiftRight<32>(MulEven(in, mul));
const Vec128<uint32_t, N> mul31{_mm_shuffle_epi32(mul.raw, 0x31)};
const auto out31 = BitCast(d32, MulEven(in31, mul31));
const Vec128<uint32_t, N> out = OddEven(out31, BitCast(d32, out20));
return IfThenElse(bits == Zero(d32), in, out);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, 1> operator>>(const Vec128<uint32_t, 1> in,
const Vec128<uint32_t, 1> bits) {
const auto bits32 =
Combine(Full64<uint32_t>(), Zero(Full32<uint32_t>()), bits);
return Vec128<uint32_t, 1>{_mm_srl_epi32(in.raw, bits32.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t> operator>>(const Vec128<uint64_t> v,
const Vec128<uint64_t> bits) {
const DFromV<decltype(v)> d;
const Vec128<uint64_t> out0{_mm_srl_epi64(v.raw, bits.raw)};
const __m128i bits1 = _mm_unpackhi_epi64(bits.raw, bits.raw);
const Vec128<uint64_t> out1{_mm_srl_epi64(v.raw, bits1)};
return ConcatUpperLower(d, out1, out0);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<uint64_t> operator>>(const Vec64<uint64_t> v,
const Vec64<uint64_t> bits) {
return Vec64<uint64_t>{_mm_srl_epi64(v.raw, bits.raw)};
}
namespace detail {
template <class DI, class V>
inline __attribute__((always_inline)) V SignedShr(const DI di, const V v, const V count_i) {
const RebindToUnsigned<DI> du;
const auto count = BitCast(du, count_i);
const auto sign = BroadcastSignBit(v);
const auto abs = BitCast(du, v ^ sign);
return BitCast(di, abs >> count) ^ sign;
}
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> operator>>(Vec128<int16_t, N> v,
Vec128<int16_t, N> bits) {
const DFromV<decltype(v)> d;
return detail::SignedShr(d, v, bits);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec16<int16_t> operator>>(Vec16<int16_t> v, Vec16<int16_t> bits) {
const auto bits16 = And(bits, Vec16<int16_t>{_mm_set_epi64x(0, 0xFFFF)});
return Vec16<int16_t>{_mm_sra_epi16(v.raw, bits16.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> operator>>(Vec128<int8_t, N> v,
Vec128<int8_t, N> bits) {
const DFromV<decltype(v)> d;
return detail::SignedShr(d, v, bits);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, 1> operator>>(Vec128<int8_t, 1> v,
Vec128<int8_t, 1> bits) {
const DFromV<decltype(v)> d;
const Rebind<int16_t, decltype(d)> di16;
const Twice<decltype(d)> dt;
const auto vi16 = ShiftRight<8>(BitCast(di16, Combine(dt, v, v)));
const Vec16<uint16_t> bits8 =
And(Vec16<uint16_t>{bits.raw}, Vec16<uint16_t>{_mm_set_epi64x(0, 0xFF)});
return Vec128<int8_t, 1>{_mm_sra_epi16(vi16.raw, bits8.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> operator>>(Vec128<int32_t, N> v,
Vec128<int32_t, N> bits) {
const DFromV<decltype(v)> d;
return detail::SignedShr(d, v, bits);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<int32_t> operator>>(Vec32<int32_t> v, Vec32<int32_t> bits) {
const auto bits32 = Combine(Full64<int32_t>(), Zero(Full32<int32_t>()), bits);
return Vec32<int32_t>{_mm_sra_epi32(v.raw, bits32.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> operator>>(Vec128<int64_t, N> v,
Vec128<int64_t, N> bits) {
const DFromV<decltype(v)> d;
return detail::SignedShr(d, v, bits);
}
inline __attribute__((always_inline)) Vec128<uint64_t> MulEven(Vec128<uint64_t> a, Vec128<uint64_t> b) {
const DFromV<decltype(a)> d;
alignas(16) uint64_t mul[2];
mul[0] = Mul128(GetLane(a), GetLane(b), &mul[1]);
return Load(d, mul);
}
inline __attribute__((always_inline)) Vec128<uint64_t> MulOdd(Vec128<uint64_t> a, Vec128<uint64_t> b) {
const DFromV<decltype(a)> d;
const Half<decltype(d)> d2;
alignas(16) uint64_t mul[2];
const uint64_t a1 = GetLane(UpperHalf(d2, a));
const uint64_t b1 = GetLane(UpperHalf(d2, b));
mul[0] = Mul128(a1, b1, &mul[1]);
return Load(d, mul);
}
template <class D32, hwy::EnableIf<IsSame<TFromD<D32>, float>()>* = nullptr,
class V16 = VFromD<Repartition<bfloat16_t, D32>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D32> WidenMulPairwiseAdd(D32 df32, V16 a, V16 b) {
const RebindToUnsigned<decltype(df32)> du32;
using VU32 = VFromD<decltype(du32)>;
const VU32 odd = Set(du32, 0xFFFF0000u);
const VU32 ae = ShiftLeft<16>(BitCast(du32, a));
const VU32 ao = And(BitCast(du32, a), odd);
const VU32 be = ShiftLeft<16>(BitCast(du32, b));
const VU32 bo = And(BitCast(du32, b), odd);
return MulAdd(BitCast(df32, ae), BitCast(df32, be),
Mul(BitCast(df32, ao), BitCast(df32, bo)));
}
template <class D32, hwy::EnableIf<IsSame<TFromD<D32>, int32_t>()>* = nullptr, hwy::EnableIf<D32::kPrivateLanes * sizeof(TFromD<D32>) <= 16>* = nullptr,
class V16 = VFromD<RepartitionToNarrow<D32>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D32> WidenMulPairwiseAdd(D32 , V16 a, V16 b) {
return VFromD<D32>{_mm_madd_epi16(a.raw, b.raw)};
}
template <class D32, hwy::EnableIf<IsSame<TFromD<D32>, float>()>* = nullptr,
class V16 = VFromD<Repartition<bfloat16_t, D32>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D32> ReorderWidenMulAccumulate(D32 df32, V16 a, V16 b,
const VFromD<D32> sum0,
VFromD<D32>& sum1) {
const RebindToUnsigned<decltype(df32)> du32;
using VU32 = VFromD<decltype(du32)>;
const VU32 odd = Set(du32, 0xFFFF0000u);
const VU32 ae = ShiftLeft<16>(BitCast(du32, a));
const VU32 ao = And(BitCast(du32, a), odd);
const VU32 be = ShiftLeft<16>(BitCast(du32, b));
const VU32 bo = And(BitCast(du32, b), odd);
sum1 = MulAdd(BitCast(df32, ao), BitCast(df32, bo), sum1);
return MulAdd(BitCast(df32, ae), BitCast(df32, be), sum0);
}
template <class D32, hwy::EnableIf<IsSame<TFromD<D32>, int32_t>()>* = nullptr, hwy::EnableIf<D32::kPrivateLanes * sizeof(TFromD<D32>) <= 16>* = nullptr,
class V16 = VFromD<RepartitionToNarrow<D32>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D32> ReorderWidenMulAccumulate(D32 d, V16 a, V16 b,
const VFromD<D32> sum0,
VFromD<D32>& ) {
(void)d;
return sum0 + WidenMulPairwiseAdd(d, a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> RearrangeToOddPlusEven(const Vec128<int32_t, N> sum0,
Vec128<int32_t, N> ) {
return sum0;
}
template <class VW>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VW RearrangeToOddPlusEven(const VW sum0, const VW sum1) {
return Add(sum0, sum1);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<uint8_t, D>> v) {
const __m128i zero = _mm_setzero_si128();
return VFromD<D>{_mm_unpacklo_epi8(v.raw, zero)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<uint16_t, D>> v) {
return VFromD<D>{_mm_unpacklo_epi16(v.raw, _mm_setzero_si128())};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<uint32_t, D>> v) {
return VFromD<D>{_mm_unpacklo_epi32(v.raw, _mm_setzero_si128())};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<uint8_t, D>> v) {
const __m128i zero = _mm_setzero_si128();
const __m128i u16 = _mm_unpacklo_epi8(v.raw, zero);
return VFromD<D>{_mm_unpacklo_epi16(u16, zero)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D d, VFromD<Rebind<uint8_t, D>> v) {
const Rebind<uint32_t, decltype(d)> du32;
return PromoteTo(d, PromoteTo(du32, v));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D d, VFromD<Rebind<uint16_t, D>> v) {
const Rebind<uint32_t, decltype(d)> du32;
return PromoteTo(d, PromoteTo(du32, v));
}
template <class D, class V, hwy::EnableIf<IsSigned<TFromD<D> >() && !IsFloat<TFromD<D> >() && !IsSpecialFloat<TFromD<D> >()>* = nullptr, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
hwy::EnableIf<(sizeof(TFromD<D>) > sizeof(TFromV<V>))>* = nullptr,
hwy::EnableIf<(D::kPrivateLanes == DFromV<V>::kPrivateLanes)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D di, V v) {
const RebindToUnsigned<decltype(di)> du;
return BitCast(di, PromoteTo(du, v));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<int8_t, D>> v) {
return ShiftRight<8>(VFromD<D>{_mm_unpacklo_epi8(v.raw, v.raw)});
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<int16_t, D>> v) {
return ShiftRight<16>(VFromD<D>{_mm_unpacklo_epi16(v.raw, v.raw)});
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<int32_t, D>> v) {
return ShiftRight<32>(VFromD<D>{_mm_unpacklo_epi32(v.raw, v.raw)});
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<int8_t, D>> v) {
const __m128i x2 = _mm_unpacklo_epi8(v.raw, v.raw);
const __m128i x4 = _mm_unpacklo_epi16(x2, x2);
return ShiftRight<24>(VFromD<D>{x4});
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D d, VFromD<Rebind<int8_t, D>> v) {
const Repartition<int32_t, decltype(d)> di32;
const Half<decltype(di32)> dh_i32;
const VFromD<decltype(di32)> x4{PromoteTo(dh_i32, v).raw};
const VFromD<decltype(di32)> s4{
_mm_shufflelo_epi16(x4.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
return ZipLower(d, x4, s4);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D d, VFromD<Rebind<int16_t, D>> v) {
const Repartition<int32_t, decltype(d)> di32;
const Half<decltype(di32)> dh_i32;
const VFromD<decltype(di32)> x2{PromoteTo(dh_i32, v).raw};
const VFromD<decltype(di32)> s2{
_mm_shufflelo_epi16(x2.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
return ZipLower(d, x2, s2);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> PromoteTo(D df32, VFromD<Rebind<float16_t, D>> v) {
const RebindToSigned<decltype(df32)> di32;
const RebindToUnsigned<decltype(df32)> du32;
const auto bits16 = PromoteTo(du32, VFromD<Rebind<uint16_t, D>>{v.raw});
const auto sign = ShiftRight<15>(bits16);
const auto biased_exp = ShiftRight<10>(bits16) & Set(du32, 0x1F);
const auto mantissa = bits16 & Set(du32, 0x3FF);
const auto subnormal =
BitCast(du32, ConvertTo(df32, BitCast(di32, mantissa)) *
Set(df32, 1.0f / 16384 / 1024));
const auto biased_exp32 = biased_exp + Set(du32, 127 - 15);
const auto mantissa32 = ShiftLeft<23 - 10>(mantissa);
const auto normal = ShiftLeft<23>(biased_exp32) | mantissa32;
const auto bits32 = IfThenElse(biased_exp == Zero(du32), subnormal, normal);
return BitCast(df32, ShiftLeft<31>(sign) | bits32);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D df32, VFromD<Rebind<bfloat16_t, D>> v) {
const Rebind<uint16_t, decltype(df32)> du16;
const RebindToSigned<decltype(df32)> di32;
return BitCast(df32, ShiftLeft<16>(PromoteTo(di32, BitCast(du16, v))));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<float, D>> v) {
return VFromD<D>{_mm_cvtps_pd(v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<int32_t, D>> v) {
return VFromD<D>{_mm_cvtepi32_pd(v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int32_t, D>> v) {
return VFromD<D>{_mm_packs_epi32(v.raw, v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int32_t, D>> v) {
const Rebind<int32_t, D> di32;
const auto zero_if_neg = AndNot(ShiftRight<31>(v), v);
const auto too_big = VecFromMask(di32, Gt(v, Set(di32, 0xFFFF)));
const auto clamped = Or(zero_if_neg, too_big);
const Rebind<uint16_t, decltype(di32)> du16;
const RebindToSigned<decltype(du16)> di16;
return BitCast(du16, DemoteTo(di16, ShiftRight<16>(ShiftLeft<16>(clamped))));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D du16, VFromD<Rebind<uint32_t, D>> v) {
const DFromV<decltype(v)> du32;
const RebindToSigned<decltype(du32)> di32;
const auto too_big =
VecFromMask(di32, Gt(BitCast(di32, ShiftRight<16>(v)), Zero(di32)));
const auto clamped = Or(BitCast(di32, v), too_big);
const RebindToSigned<decltype(du16)> di16;
return BitCast(du16, DemoteTo(di16, ShiftRight<16>(ShiftLeft<16>(clamped))));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int32_t, D>> v) {
const __m128i i16 = _mm_packs_epi32(v.raw, v.raw);
return VFromD<D>{_mm_packus_epi16(i16, i16)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int16_t, D>> v) {
return VFromD<D>{_mm_packus_epi16(v.raw, v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int32_t, D>> v) {
const __m128i i16 = _mm_packs_epi32(v.raw, v.raw);
return VFromD<D>{_mm_packs_epi16(i16, i16)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int16_t, D>> v) {
return VFromD<D>{_mm_packs_epi16(v.raw, v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D du8, VFromD<Rebind<uint32_t, D>> v) {
const DFromV<decltype(v)> du32;
const RebindToSigned<decltype(du32)> di32;
const auto max_i32 = Set(du32, 0x7FFFFFFFu);
const Repartition<uint8_t, decltype(du32)> du32_as_du8;
const auto clamped = BitCast(
di32, Min(BitCast(du32_as_du8, v), BitCast(du32_as_du8, max_i32)));
return DemoteTo(du8, clamped);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D du8, VFromD<Rebind<uint16_t, D>> v) {
const DFromV<decltype(v)> du16;
const RebindToSigned<decltype(du16)> di16;
const auto max_i16 = Set(du16, 0x7FFF);
const Repartition<uint8_t, decltype(du16)> du16_as_du8;
const auto clamped = BitCast(
di16, Min(BitCast(du16_as_du8, v), BitCast(du16_as_du8, max_i16)));
return DemoteTo(du8, clamped);
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmain"
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D df16, VFromD<Rebind<float, D>> v) {
const RebindToUnsigned<decltype(df16)> du16;
const Rebind<uint32_t, decltype(df16)> du;
const RebindToSigned<decltype(du)> di;
const auto bits32 = BitCast(du, v);
const auto sign = ShiftRight<31>(bits32);
const auto biased_exp32 = ShiftRight<23>(bits32) & Set(du, 0xFF);
const auto mantissa32 = bits32 & Set(du, 0x7FFFFF);
const auto k15 = Set(di, 15);
const auto exp = Min(BitCast(di, biased_exp32) - Set(di, 127), k15);
const auto is_tiny = exp < Set(di, -24);
const auto is_subnormal = exp < Set(di, -14);
const auto biased_exp16 =
BitCast(du, IfThenZeroElse(is_subnormal, exp + k15));
const auto sub_exp = BitCast(du, Set(di, -14) - exp);
const auto sub_m = (Set(du, 1) << (Set(du, 10) - sub_exp)) +
(mantissa32 >> (Set(du, 13) + sub_exp));
const auto mantissa16 = IfThenElse(RebindMask(du, is_subnormal), sub_m,
ShiftRight<13>(mantissa32));
const auto sign16 = ShiftLeft<15>(sign);
const auto normal16 = sign16 | ShiftLeft<10>(biased_exp16) | mantissa16;
const auto bits16 = IfThenZeroElse(is_tiny, BitCast(di, normal16));
return BitCast(df16, DemoteTo(du16, bits16));
}
#pragma GCC diagnostic pop
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, bfloat16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D dbf16, VFromD<Rebind<float, D>> v) {
const Rebind<int32_t, decltype(dbf16)> di32;
const Rebind<uint32_t, decltype(dbf16)> du32;
const Rebind<uint16_t, decltype(dbf16)> du16;
const auto bits_in_32 = BitCast(di32, ShiftRight<16>(BitCast(du32, v)));
return BitCast(dbf16, DemoteTo(du16, bits_in_32));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, bfloat16_t>()>* = nullptr,
class V32 = VFromD<Repartition<float, D>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dbf16, V32 a, V32 b) {
const RebindToUnsigned<decltype(dbf16)> du16;
const Repartition<uint32_t, decltype(dbf16)> du32;
const VFromD<decltype(du32)> b_in_even = ShiftRight<16>(BitCast(du32, b));
return BitCast(dbf16, OddEven(BitCast(du16, a), BitCast(du16, b_in_even)));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<int16_t> ReorderDemote2To(D dn, Vec32<int32_t> a,
Vec32<int32_t> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<int16_t> ReorderDemote2To(D , Vec64<int32_t> a,
Vec64<int32_t> b) {
return Vec64<int16_t>{_mm_shuffle_epi32(_mm_packs_epi32(a.raw, b.raw),
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t> ReorderDemote2To(D , Vec128<int32_t> a,
Vec128<int32_t> b) {
return Vec128<int16_t>{_mm_packs_epi32(a.raw, b.raw)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<uint16_t> ReorderDemote2To(D dn, Vec32<int32_t> a,
Vec32<int32_t> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<uint16_t> ReorderDemote2To(D dn, Vec64<int32_t> a,
Vec64<int32_t> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t> ReorderDemote2To(D dn, Vec128<int32_t> a,
Vec128<int32_t> b) {
const Half<decltype(dn)> dnh;
const auto u16_a = DemoteTo(dnh, a);
const auto u16_b = DemoteTo(dnh, b);
return Combine(dn, u16_b, u16_a);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, Vec128<uint32_t> a,
Vec128<uint32_t> b) {
const DFromV<decltype(a)> du32;
const RebindToSigned<decltype(du32)> di32;
const auto max_i32 = Set(du32, 0x7FFFFFFFu);
const Repartition<uint8_t, decltype(du32)> du32_as_du8;
const auto clamped_a = BitCast(
di32, Min(BitCast(du32_as_du8, a), BitCast(du32_as_du8, max_i32)));
const auto clamped_b = BitCast(
di32, Min(BitCast(du32_as_du8, b), BitCast(du32_as_du8, max_i32)));
return ReorderDemote2To(dn, clamped_a, clamped_b);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<uint32_t, D>> a,
VFromD<Repartition<uint32_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<int16_t, D>> a,
VFromD<Repartition<int16_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<int8_t> ReorderDemote2To(D , Vec64<int16_t> a,
Vec64<int16_t> b) {
return Vec64<int8_t>{_mm_shuffle_epi32(_mm_packs_epi16(a.raw, b.raw),
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t> ReorderDemote2To(D , Vec128<int16_t> a,
Vec128<int16_t> b) {
return Vec128<int8_t>{_mm_packs_epi16(a.raw, b.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<int16_t, D>> a,
VFromD<Repartition<int16_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<uint8_t> ReorderDemote2To(D , Vec64<int16_t> a,
Vec64<int16_t> b) {
return Vec64<uint8_t>{_mm_shuffle_epi32(_mm_packus_epi16(a.raw, b.raw),
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t> ReorderDemote2To(D , Vec128<int16_t> a,
Vec128<int16_t> b) {
return Vec128<uint8_t>{_mm_packus_epi16(a.raw, b.raw)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, Vec128<uint16_t> a,
Vec128<uint16_t> b) {
const DFromV<decltype(a)> du16;
const RebindToSigned<decltype(du16)> di16;
const auto max_i16 = Set(du16, 0x7FFFu);
const Repartition<uint8_t, decltype(du16)> du16_as_du8;
const auto clamped_a = BitCast(
di16, Min(BitCast(du16_as_du8, a), BitCast(du16_as_du8, max_i16)));
const auto clamped_b = BitCast(
di16, Min(BitCast(du16_as_du8, b), BitCast(du16_as_du8, max_i16)));
return ReorderDemote2To(dn, clamped_a, clamped_b);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<uint16_t, D>> a,
VFromD<Repartition<uint16_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<!hwy::IsFloat<TFromD<D> >() && !hwy::IsSpecialFloat<TFromD<D> >()>* = nullptr,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr,
hwy::EnableIf<sizeof(TFromV<V>) == (sizeof(TFromD<D>) * 2)>* = nullptr,
hwy::EnableIf<(D::kPrivateLanes == DFromV<V>::kPrivateLanes * 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> OrderedDemote2To(D d, V a, V b) {
return ReorderDemote2To(d, a, b);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, bfloat16_t>()>* = nullptr, class V32 = VFromD<Repartition<float, D>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> OrderedDemote2To(D dbf16, V32 a, V32 b) {
const RebindToUnsigned<decltype(dbf16)> du16;
return BitCast(dbf16, ConcatOdd(du16, BitCast(du16, b), BitCast(du16, a)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<double, D>> v) {
return VFromD<D>{_mm_cvtpd_ps(v.raw)};
}
namespace detail {
template <class D>
inline __attribute__((always_inline)) VFromD<D> ClampF64ToI32Max(D d, VFromD<D> v) {
return Min(v, Set(d, 2147483647.0));
}
template <class DI, class DF = RebindToFloat<DI>>
inline __attribute__((always_inline)) VFromD<DI> FixConversionOverflow(
DI di, VFromD<DF> original, decltype(Zero(DI()).raw) converted_raw) {
const VFromD<DI> converted{converted_raw};
const VFromD<DI> sign_wrong = AndNot(BitCast(di, original), converted);
const RebindToUnsigned<DI> du;
const VFromD<DI> mask = BroadcastSignBit(sign_wrong);
const VFromD<DI> max = BitCast(di, ShiftRight<1>(BitCast(du, mask)));
return IfVecThenElse(mask, max, converted);
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr,
class DF = Rebind<double, D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<DF> v) {
const VFromD<DF> clamped = detail::ClampF64ToI32Max(DF(), v);
return VFromD<D>{_mm_cvttpd_epi32(clamped.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> U8FromU32(const Vec128<uint32_t, N> v) {
const RebindToSigned<DFromV<decltype(v)>> di32;
const Rebind<uint8_t, decltype(di32)> du8;
return DemoteTo(du8, BitCast(di32, v));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t> MulFixedPoint15(const Vec128<int16_t> a,
const Vec128<int16_t> b) {
const DFromV<decltype(a)> d;
const Repartition<int32_t, decltype(d)> di32;
auto lo_product = a * b;
auto hi_product = MulHigh(a, b);
const VFromD<decltype(di32)> i32_product_lo{
_mm_unpacklo_epi16(lo_product.raw, hi_product.raw)};
const VFromD<decltype(di32)> i32_product_hi{
_mm_unpackhi_epi16(lo_product.raw, hi_product.raw)};
const auto round_up_incr = Set(di32, 0x4000);
return ReorderDemote2To(d, ShiftRight<15>(i32_product_lo + round_up_incr),
ShiftRight<15>(i32_product_hi + round_up_incr));
}
template <size_t N, hwy::EnableIf<N * sizeof(int16_t) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> MulFixedPoint15(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
const DFromV<decltype(a)> d;
const Rebind<int32_t, decltype(d)> di32;
const auto lo_product = a * b;
const auto hi_product = MulHigh(a, b);
const VFromD<decltype(di32)> i32_product{
_mm_unpacklo_epi16(lo_product.raw, hi_product.raw)};
return DemoteTo(d, ShiftRight<15>(i32_product + Set(di32, 0x4000)));
}
template <typename From, class DTo, hwy::EnableIf<(DTo::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DTo> TruncateTo(DTo , Vec128<From, 1> v) {
const Repartition<TFromD<DTo>, DFromV<decltype(v)>> dto;
return VFromD<DTo>{BitCast(dto, v).raw};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec16<uint8_t> TruncateTo(D d, Vec128<uint64_t> v) {
const Vec128<uint8_t, 1> lo{v.raw};
const Vec128<uint8_t, 1> hi{_mm_unpackhi_epi64(v.raw, v.raw)};
return Combine(d, hi, lo);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<uint16_t> TruncateTo(D d, Vec128<uint64_t> v) {
const Vec128<uint16_t, 1> lo{v.raw};
const Vec128<uint16_t, 1> hi{_mm_unpackhi_epi64(v.raw, v.raw)};
return Combine(d, hi, lo);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<uint32_t> TruncateTo(D , Vec128<uint64_t> v) {
return Vec64<uint32_t>{_mm_shuffle_epi32(v.raw, 0x88)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TruncateTo(D , VFromD<Rebind<uint32_t, D>> v) {
const DFromV<decltype(v)> du32;
const RebindToSigned<decltype(du32)> di32;
const Rebind<uint8_t, decltype(di32)> du8;
return DemoteTo(du8, BitCast(di32, ShiftRight<24>(ShiftLeft<24>(v))));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TruncateTo(D , VFromD<Rebind<uint32_t, D>> v) {
const DFromV<decltype(v)> du32;
const RebindToSigned<decltype(du32)> di32;
const Rebind<uint16_t, decltype(di32)> du16;
const RebindToSigned<decltype(du16)> di16;
return BitCast(
du16, DemoteTo(di16, ShiftRight<16>(BitCast(di32, ShiftLeft<16>(v)))));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TruncateTo(D , VFromD<Rebind<uint16_t, D>> v) {
const DFromV<decltype(v)> du16;
const RebindToSigned<decltype(du16)> di16;
const Rebind<uint8_t, decltype(di16)> du8;
const RebindToSigned<decltype(du8)> di8;
return BitCast(du8,
DemoteTo(di8, ShiftRight<8>(BitCast(di16, ShiftLeft<8>(v)))));
}
namespace detail {
template <class D, hwy::EnableIf<!IsSigned<TFromD<D> >()>* = nullptr>
inline __attribute__((always_inline)) VFromD<Rebind<uint64_t, D>> DemoteFromU64MaskOutResult(
D , VFromD<Rebind<uint64_t, D>> v) {
return v;
}
template <class D, hwy::EnableIf<IsSigned<TFromD<D> >() && !IsFloat<TFromD<D> >() && !IsSpecialFloat<TFromD<D> >()>* = nullptr>
inline __attribute__((always_inline)) VFromD<Rebind<uint64_t, D>> DemoteFromU64MaskOutResult(
D , VFromD<Rebind<uint64_t, D>> v) {
const DFromV<decltype(v)> du64;
return And(v,
Set(du64, static_cast<uint64_t>(hwy::HighestValue<TFromD<D>>())));
}
template <class D>
inline __attribute__((always_inline)) VFromD<Rebind<uint64_t, D>> DemoteFromU64Saturate(
D dn, VFromD<Rebind<uint64_t, D>> v) {
const Rebind<uint64_t, D> du64;
const RebindToSigned<decltype(du64)> di64;
constexpr int kShiftAmt = static_cast<int>(sizeof(TFromD<D>) * 8) -
static_cast<int>(hwy::IsSigned<TFromD<D>>());
const auto too_big = BitCast(
du64, VecFromMask(
di64, Gt(BitCast(di64, ShiftRight<kShiftAmt>(v)), Zero(di64))));
return DemoteFromU64MaskOutResult(dn, Or(v, too_big));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, class V>
inline __attribute__((always_inline)) VFromD<D> ReorderDemote2From64To32Combine(D dn, V a, V b) {
return ConcatEven(dn, BitCast(dn, b), BitCast(dn, a));
}
}
template <class D, hwy::EnableIf<((size_t{1} << sizeof(TFromD<D>)) & ((1 << 1) | (1 << 2) | (1 << 4))) != 0>* = nullptr,
hwy::EnableIf<IsSigned<TFromD<D> >() && !IsFloat<TFromD<D> >() && !IsSpecialFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D dn, VFromD<Rebind<int64_t, D>> v) {
const DFromV<decltype(v)> di64;
const RebindToUnsigned<decltype(di64)> du64;
const RebindToUnsigned<decltype(dn)> dn_u;
const auto invert_mask = BitCast(du64, BroadcastSignBit(v));
const auto saturated_vals = Xor(
invert_mask,
detail::DemoteFromU64Saturate(dn, Xor(invert_mask, BitCast(du64, v))));
return BitCast(dn, TruncateTo(dn_u, saturated_vals));
}
template <class D, hwy::EnableIf<((size_t{1} << sizeof(TFromD<D>)) & ((1 << 1) | (1 << 2) | (1 << 4))) != 0>* = nullptr,
hwy::EnableIf<!IsSigned<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D dn, VFromD<Rebind<int64_t, D>> v) {
const DFromV<decltype(v)> di64;
const RebindToUnsigned<decltype(di64)> du64;
const auto non_neg_vals = BitCast(du64, AndNot(BroadcastSignBit(v), v));
return TruncateTo(dn, detail::DemoteFromU64Saturate(dn, non_neg_vals));
}
template <class D, hwy::EnableIf<((size_t{1} << sizeof(TFromD<D>)) & ((1 << 1) | (1 << 2) | (1 << 4))) != 0>* = nullptr,
hwy::EnableIf<!IsSigned<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D dn, VFromD<Rebind<uint64_t, D>> v) {
return TruncateTo(dn, detail::DemoteFromU64Saturate(dn, v));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16 / 2>* = nullptr,
hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr, hwy::EnableIf<!hwy::IsFloat<TFromD<D> >() && !hwy::IsSpecialFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<int64_t, D>> a,
VFromD<Repartition<int64_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16 / 2>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<uint64_t, D>> a,
VFromD<Repartition<uint64_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> ReorderDemote2To(D dn, Vec128<int64_t> a,
Vec128<int64_t> b) {
const DFromV<decltype(a)> di64;
const RebindToUnsigned<decltype(di64)> du64;
const Half<decltype(dn)> dnh;
const auto invert_mask_a = BitCast(du64, BroadcastSignBit(a));
const auto invert_mask_b = BitCast(du64, BroadcastSignBit(b));
const auto saturated_a = Xor(
invert_mask_a,
detail::DemoteFromU64Saturate(dnh, Xor(invert_mask_a, BitCast(du64, a))));
const auto saturated_b = Xor(
invert_mask_b,
detail::DemoteFromU64Saturate(dnh, Xor(invert_mask_b, BitCast(du64, b))));
return ConcatEven(dn, BitCast(dn, saturated_b), BitCast(dn, saturated_a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> ReorderDemote2To(D dn, Vec128<int64_t> a,
Vec128<int64_t> b) {
const DFromV<decltype(a)> di64;
const RebindToUnsigned<decltype(di64)> du64;
const Half<decltype(dn)> dnh;
const auto saturated_a = detail::DemoteFromU64Saturate(
dnh, BitCast(du64, AndNot(BroadcastSignBit(a), a)));
const auto saturated_b = detail::DemoteFromU64Saturate(
dnh, BitCast(du64, AndNot(BroadcastSignBit(b), b)));
return ConcatEven(dn, BitCast(dn, saturated_b), BitCast(dn, saturated_a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> ReorderDemote2To(D dn, Vec128<uint64_t> a,
Vec128<uint64_t> b) {
const Half<decltype(dn)> dnh;
const auto saturated_a = detail::DemoteFromU64Saturate(dnh, a);
const auto saturated_b = detail::DemoteFromU64Saturate(dnh, b);
return ConcatEven(dn, BitCast(dn, saturated_b), BitCast(dn, saturated_a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConvertTo(D , VFromD<Rebind<int32_t, D>> v) {
return VFromD<D>{_mm_cvtepi32_ps(v.raw)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConvertTo(D df, VFromD<Rebind<uint32_t, D>> v) {
const RebindToUnsigned<decltype(df)> du32;
const RebindToSigned<decltype(df)> d32;
const auto msk_lo = Set(du32, 0xFFFF);
const auto cnst2_16_flt = Set(df, 65536.0f);
const auto v_lo = BitCast(d32, And(v, msk_lo));
const auto v_hi = BitCast(d32, ShiftRight<16>(v));
return MulAdd(cnst2_16_flt, ConvertTo(df, v_hi), ConvertTo(df, v_lo));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConvertTo(D dd, VFromD<Rebind<int64_t, D>> v) {
const Repartition<uint32_t, decltype(dd)> d32;
const Repartition<uint64_t, decltype(dd)> d64;
const auto k84_63 = Set(d64, 0x4530000080000000ULL);
const auto v_upper = BitCast(dd, ShiftRight<32>(BitCast(d64, v)) ^ k84_63);
const auto k52 = Set(d32, 0x43300000);
const auto v_lower = BitCast(dd, OddEven(k52, BitCast(d32, v)));
const auto k84_63_52 = BitCast(dd, Set(d64, 0x4530000080100000ULL));
return (v_upper - k84_63_52) + v_lower;
}
namespace detail {
template <class VW>
inline __attribute__((always_inline)) VFromD<Rebind<double, DFromV<VW>>> U64ToF64VecFast(VW w) {
const DFromV<decltype(w)> d64;
const RebindToFloat<decltype(d64)> dd;
const auto cnst2_52_dbl = Set(dd, 0x0010000000000000);
return BitCast(dd, Or(w, BitCast(d64, cnst2_52_dbl))) - cnst2_52_dbl;
}
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConvertTo(D dd, VFromD<Rebind<uint64_t, D>> v) {
const RebindToUnsigned<decltype(dd)> d64;
using VU = VFromD<decltype(d64)>;
const VU msk_lo = Set(d64, 0xFFFFFFFF);
const auto cnst2_32_dbl = Set(dd, 4294967296.0);
const VU v_lo = And(v, msk_lo);
const VU v_hi = ShiftRight<32>(v);
const auto v_lo_dbl = detail::U64ToF64VecFast(v_lo);
return MulAdd(cnst2_32_dbl, detail::U64ToF64VecFast(v_hi), v_lo_dbl);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConvertTo(D di, VFromD<Rebind<float, D>> v) {
return detail::FixConversionOverflow(di, v, _mm_cvttps_epi32(v.raw));
}
template <class DI, hwy::EnableIf<(DI::kPrivateLanes * sizeof(TFromD<DI>) > (0 ? 16 : 0))>* = nullptr,
hwy::EnableIf<IsSame<TFromD<DI>, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DI> ConvertTo(DI di, VFromD<Rebind<double, DI>> v) {
using VI = VFromD<decltype(di)>;
const RebindToUnsigned<decltype(di)> du;
using VU = VFromD<decltype(du)>;
const Repartition<uint16_t, decltype(di)> du16;
const VI k1075 = Set(di, 1075);
const VU biased_exp = ShiftRight<52>(BitCast(du, v)) & Set(du, 0x7FF);
const Repartition<int32_t, decltype(di)> di32;
const auto in_range = MaskFromVec(BitCast(
di,
VecFromMask(di32, DupEven(BitCast(di32, biased_exp)) < Set(di32, 1086))));
const VU shift_mnt = BitCast(
du, SaturatedSub(BitCast(du16, k1075), BitCast(du16, biased_exp)));
const VU shift_int = BitCast(
du, SaturatedSub(BitCast(du16, biased_exp), BitCast(du16, k1075)));
const VU mantissa = BitCast(du, v) & Set(du, (1ULL << 52) - 1);
const VU int53 = (mantissa | Set(du, 1ULL << 52)) >> shift_mnt;
const VU shifted = int53 << shift_int;
const VI sign_mask = BroadcastSignBit(BitCast(di, v));
const VI limit = Set(di, LimitsMax<int64_t>()) - sign_mask;
const VI magnitude = IfThenElse(in_range, BitCast(di, shifted), limit);
return (magnitude ^ sign_mask) - sign_mask;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> NearestInt(const Vec128<float, N> v) {
const RebindToSigned<DFromV<decltype(v)>> di;
return detail::FixConversionOverflow(di, v, _mm_cvtps_epi32(v.raw));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Round(const Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> df;
const auto max = Set(df, MantissaEnd<T>());
const auto large = CopySignToAbs(max, v);
const auto added = large + v;
const auto rounded = added - large;
return IfThenElse(Abs(v) < max, rounded, v);
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) Mask128<T, N> UseInt(const Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> d;
return Abs(v) < Set(d, MantissaEnd<T>());
}
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Trunc(const Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> df;
const RebindToSigned<decltype(df)> di;
const auto integer = ConvertTo(di, v);
const auto int_f = ConvertTo(df, integer);
return IfThenElse(detail::UseInt(v), CopySign(int_f, v), v);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Ceil(const Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> df;
const RebindToSigned<decltype(df)> di;
const auto integer = ConvertTo(di, v);
const auto int_f = ConvertTo(df, integer);
const auto neg1 = ConvertTo(df, VecFromMask(di, RebindMask(di, int_f < v)));
return IfThenElse(detail::UseInt(v), int_f - neg1, v);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Floor(const Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> df;
const RebindToSigned<decltype(df)> di;
const auto integer = ConvertTo(di, v);
const auto int_f = ConvertTo(df, integer);
const auto neg1 = ConvertTo(df, VecFromMask(di, RebindMask(di, int_f > v)));
return IfThenElse(detail::UseInt(v), int_f + neg1, v);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<float, N> IsNaN(const Vec128<float, N> v) {
return Mask128<float, N>{_mm_cmpunord_ps(v.raw, v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<double, N> IsNaN(const Vec128<double, N> v) {
return Mask128<double, N>{_mm_cmpunord_pd(v.raw, v.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> IsInf(const Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> d;
const RebindToSigned<decltype(d)> di;
const VFromD<decltype(di)> vi = BitCast(di, v);
return RebindMask(d, Eq(Add(vi, vi), Set(di, hwy::MaxExponentTimes2<T>())));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> IsFinite(const Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const RebindToSigned<decltype(d)> di;
const VFromD<decltype(du)> vu = BitCast(du, v);
const VFromD<decltype(di)> exp =
BitCast(di, ShiftRight<hwy::MantissaBits<T>() + 1>(ShiftLeft<1>(vu)));
return RebindMask(d, Lt(exp, Set(di, hwy::MaxExponentField<T>())));
}
namespace detail {
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
inline __attribute__((always_inline)) MFromD<D> LoadMaskBits128(D d, uint64_t mask_bits) {
const RebindToUnsigned<decltype(d)> du;
const VFromD<D> vbits{_mm_cvtsi32_si128(static_cast<int>(mask_bits))};
__m128i unpacked_vbits = _mm_unpacklo_epi8(vbits.raw, vbits.raw);
unpacked_vbits = _mm_unpacklo_epi16(unpacked_vbits, unpacked_vbits);
const VFromD<decltype(du)> rep8{
_mm_unpacklo_epi32(unpacked_vbits, unpacked_vbits)};
alignas(16) static constexpr uint8_t kBit[16] = {1, 2, 4, 8, 16, 32, 64, 128,
1, 2, 4, 8, 16, 32, 64, 128};
return RebindMask(d, TestBit(rep8, LoadDup128(du, kBit)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
inline __attribute__((always_inline)) MFromD<D> LoadMaskBits128(D d, uint64_t mask_bits) {
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint16_t kBit[8] = {1, 2, 4, 8, 16, 32, 64, 128};
const auto vmask_bits = Set(du, static_cast<uint16_t>(mask_bits));
return RebindMask(d, TestBit(vmask_bits, Load(du, kBit)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
inline __attribute__((always_inline)) MFromD<D> LoadMaskBits128(D d, uint64_t mask_bits) {
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint32_t kBit[8] = {1, 2, 4, 8};
const auto vmask_bits = Set(du, static_cast<uint32_t>(mask_bits));
return RebindMask(d, TestBit(vmask_bits, Load(du, kBit)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
inline __attribute__((always_inline)) MFromD<D> LoadMaskBits128(D d, uint64_t mask_bits) {
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint64_t kBit[8] = {1, 2};
return RebindMask(d, TestBit(Set(du, mask_bits), Load(du, kBit)));
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> LoadMaskBits(D d, const uint8_t* __restrict__ bits) {
constexpr size_t kN = MaxLanes(d);
uint64_t mask_bits = 0;
constexpr size_t kNumBytes = (kN + 7) / 8;
CopyBytes<kNumBytes>(bits, &mask_bits);
if (kN < 8) {
mask_bits &= (1ull << kN) - 1;
}
return detail::LoadMaskBits128(d, mask_bits);
}
template <typename T>
struct CompressIsPartition {
enum { value = (sizeof(T) != 1) };
};
namespace detail {
constexpr inline __attribute__((always_inline)) uint64_t U64FromInt(int mask_bits) {
return static_cast<uint64_t>(static_cast<unsigned>(mask_bits));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) uint64_t BitsFromMask(hwy::SizeTag<1> ,
const Mask128<T, N> mask) {
const Simd<T, N, 0> d;
const auto sign_bits = BitCast(d, VecFromMask(d, mask)).raw;
return U64FromInt(_mm_movemask_epi8(sign_bits));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) uint64_t BitsFromMask(hwy::SizeTag<2> ,
const Mask128<T, N> mask) {
const auto sign_bits = _mm_packs_epi16(mask.raw, _mm_setzero_si128());
return U64FromInt(_mm_movemask_epi8(sign_bits));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) uint64_t BitsFromMask(hwy::SizeTag<4> , Mask128<T, N> mask) {
const Simd<T, N, 0> d;
const Simd<float, N, 0> df;
const auto sign_bits = BitCast(df, VecFromMask(d, mask));
return U64FromInt(_mm_movemask_ps(sign_bits.raw));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) uint64_t BitsFromMask(hwy::SizeTag<8> , Mask128<T, N> mask) {
const Simd<T, N, 0> d;
const Simd<double, N, 0> df;
const auto sign_bits = BitCast(df, VecFromMask(d, mask));
return U64FromInt(_mm_movemask_pd(sign_bits.raw));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) uint64_t BitsFromMask(const Mask128<T, N> mask) {
return OnlyActive<T, N>(BitsFromMask(hwy::SizeTag<sizeof(T)>(), mask));
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t StoreMaskBits(D d, MFromD<D> mask, uint8_t* bits) {
constexpr size_t kNumBytes = (MaxLanes(d) + 7) / 8;
const uint64_t mask_bits = detail::BitsFromMask(mask);
CopyBytes<kNumBytes>(&mask_bits, bits);
return kNumBytes;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) bool AllFalse(D , MFromD<D> mask) {
return detail::BitsFromMask(mask) == 0;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) bool AllTrue(D d, MFromD<D> mask) {
constexpr uint64_t kAllBits = (1ull << MaxLanes(d)) - 1;
return detail::BitsFromMask(mask) == kAllBits;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CountTrue(D , MFromD<D> mask) {
return PopCount(detail::BitsFromMask(mask));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t FindKnownFirstTrue(D , MFromD<D> mask) {
return Num0BitsBelowLS1Bit_Nonzero32(
static_cast<uint32_t>(detail::BitsFromMask(mask)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) intptr_t FindFirstTrue(D , MFromD<D> mask) {
const uint32_t mask_bits = static_cast<uint32_t>(detail::BitsFromMask(mask));
return mask_bits ? intptr_t(Num0BitsBelowLS1Bit_Nonzero32(mask_bits)) : -1;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t FindKnownLastTrue(D , MFromD<D> mask) {
return 31 - Num0BitsAboveMS1Bit_Nonzero32(
static_cast<uint32_t>(detail::BitsFromMask(mask)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) intptr_t FindLastTrue(D , MFromD<D> mask) {
const uint32_t mask_bits = static_cast<uint32_t>(detail::BitsFromMask(mask));
return mask_bits ? intptr_t(31 - Num0BitsAboveMS1Bit_Nonzero32(mask_bits))
: -1;
}
namespace detail {
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromBits128(D d, uint64_t mask_bits) {
do { } while (0);
const Rebind<uint8_t, decltype(d)> d8;
const Twice<decltype(d8)> d8t;
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint8_t table[2048] = {
0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
2, 0, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
4, 0, 2, 6, 8, 10, 12, 14, 0, 4, 2, 6, 8, 10, 12, 14,
2, 4, 0, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
6, 0, 2, 4, 8, 10, 12, 14, 0, 6, 2, 4, 8, 10, 12, 14,
2, 6, 0, 4, 8, 10, 12, 14, 0, 2, 6, 4, 8, 10, 12, 14,
4, 6, 0, 2, 8, 10, 12, 14, 0, 4, 6, 2, 8, 10, 12, 14,
2, 4, 6, 0, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
8, 0, 2, 4, 6, 10, 12, 14, 0, 8, 2, 4, 6, 10, 12, 14,
2, 8, 0, 4, 6, 10, 12, 14, 0, 2, 8, 4, 6, 10, 12, 14,
4, 8, 0, 2, 6, 10, 12, 14, 0, 4, 8, 2, 6, 10, 12, 14,
2, 4, 8, 0, 6, 10, 12, 14, 0, 2, 4, 8, 6, 10, 12, 14,
6, 8, 0, 2, 4, 10, 12, 14, 0, 6, 8, 2, 4, 10, 12, 14,
2, 6, 8, 0, 4, 10, 12, 14, 0, 2, 6, 8, 4, 10, 12, 14,
4, 6, 8, 0, 2, 10, 12, 14, 0, 4, 6, 8, 2, 10, 12, 14,
2, 4, 6, 8, 0, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
10, 0, 2, 4, 6, 8, 12, 14, 0, 10, 2, 4, 6, 8, 12, 14,
2, 10, 0, 4, 6, 8, 12, 14, 0, 2, 10, 4, 6, 8, 12, 14,
4, 10, 0, 2, 6, 8, 12, 14, 0, 4, 10, 2, 6, 8, 12, 14,
2, 4, 10, 0, 6, 8, 12, 14, 0, 2, 4, 10, 6, 8, 12, 14,
6, 10, 0, 2, 4, 8, 12, 14, 0, 6, 10, 2, 4, 8, 12, 14,
2, 6, 10, 0, 4, 8, 12, 14, 0, 2, 6, 10, 4, 8, 12, 14,
4, 6, 10, 0, 2, 8, 12, 14, 0, 4, 6, 10, 2, 8, 12, 14,
2, 4, 6, 10, 0, 8, 12, 14, 0, 2, 4, 6, 10, 8, 12, 14,
8, 10, 0, 2, 4, 6, 12, 14, 0, 8, 10, 2, 4, 6, 12, 14,
2, 8, 10, 0, 4, 6, 12, 14, 0, 2, 8, 10, 4, 6, 12, 14,
4, 8, 10, 0, 2, 6, 12, 14, 0, 4, 8, 10, 2, 6, 12, 14,
2, 4, 8, 10, 0, 6, 12, 14, 0, 2, 4, 8, 10, 6, 12, 14,
6, 8, 10, 0, 2, 4, 12, 14, 0, 6, 8, 10, 2, 4, 12, 14,
2, 6, 8, 10, 0, 4, 12, 14, 0, 2, 6, 8, 10, 4, 12, 14,
4, 6, 8, 10, 0, 2, 12, 14, 0, 4, 6, 8, 10, 2, 12, 14,
2, 4, 6, 8, 10, 0, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
12, 0, 2, 4, 6, 8, 10, 14, 0, 12, 2, 4, 6, 8, 10, 14,
2, 12, 0, 4, 6, 8, 10, 14, 0, 2, 12, 4, 6, 8, 10, 14,
4, 12, 0, 2, 6, 8, 10, 14, 0, 4, 12, 2, 6, 8, 10, 14,
2, 4, 12, 0, 6, 8, 10, 14, 0, 2, 4, 12, 6, 8, 10, 14,
6, 12, 0, 2, 4, 8, 10, 14, 0, 6, 12, 2, 4, 8, 10, 14,
2, 6, 12, 0, 4, 8, 10, 14, 0, 2, 6, 12, 4, 8, 10, 14,
4, 6, 12, 0, 2, 8, 10, 14, 0, 4, 6, 12, 2, 8, 10, 14,
2, 4, 6, 12, 0, 8, 10, 14, 0, 2, 4, 6, 12, 8, 10, 14,
8, 12, 0, 2, 4, 6, 10, 14, 0, 8, 12, 2, 4, 6, 10, 14,
2, 8, 12, 0, 4, 6, 10, 14, 0, 2, 8, 12, 4, 6, 10, 14,
4, 8, 12, 0, 2, 6, 10, 14, 0, 4, 8, 12, 2, 6, 10, 14,
2, 4, 8, 12, 0, 6, 10, 14, 0, 2, 4, 8, 12, 6, 10, 14,
6, 8, 12, 0, 2, 4, 10, 14, 0, 6, 8, 12, 2, 4, 10, 14,
2, 6, 8, 12, 0, 4, 10, 14, 0, 2, 6, 8, 12, 4, 10, 14,
4, 6, 8, 12, 0, 2, 10, 14, 0, 4, 6, 8, 12, 2, 10, 14,
2, 4, 6, 8, 12, 0, 10, 14, 0, 2, 4, 6, 8, 12, 10, 14,
10, 12, 0, 2, 4, 6, 8, 14, 0, 10, 12, 2, 4, 6, 8, 14,
2, 10, 12, 0, 4, 6, 8, 14, 0, 2, 10, 12, 4, 6, 8, 14,
4, 10, 12, 0, 2, 6, 8, 14, 0, 4, 10, 12, 2, 6, 8, 14,
2, 4, 10, 12, 0, 6, 8, 14, 0, 2, 4, 10, 12, 6, 8, 14,
6, 10, 12, 0, 2, 4, 8, 14, 0, 6, 10, 12, 2, 4, 8, 14,
2, 6, 10, 12, 0, 4, 8, 14, 0, 2, 6, 10, 12, 4, 8, 14,
4, 6, 10, 12, 0, 2, 8, 14, 0, 4, 6, 10, 12, 2, 8, 14,
2, 4, 6, 10, 12, 0, 8, 14, 0, 2, 4, 6, 10, 12, 8, 14,
8, 10, 12, 0, 2, 4, 6, 14, 0, 8, 10, 12, 2, 4, 6, 14,
2, 8, 10, 12, 0, 4, 6, 14, 0, 2, 8, 10, 12, 4, 6, 14,
4, 8, 10, 12, 0, 2, 6, 14, 0, 4, 8, 10, 12, 2, 6, 14,
2, 4, 8, 10, 12, 0, 6, 14, 0, 2, 4, 8, 10, 12, 6, 14,
6, 8, 10, 12, 0, 2, 4, 14, 0, 6, 8, 10, 12, 2, 4, 14,
2, 6, 8, 10, 12, 0, 4, 14, 0, 2, 6, 8, 10, 12, 4, 14,
4, 6, 8, 10, 12, 0, 2, 14, 0, 4, 6, 8, 10, 12, 2, 14,
2, 4, 6, 8, 10, 12, 0, 14, 0, 2, 4, 6, 8, 10, 12, 14,
14, 0, 2, 4, 6, 8, 10, 12, 0, 14, 2, 4, 6, 8, 10, 12,
2, 14, 0, 4, 6, 8, 10, 12, 0, 2, 14, 4, 6, 8, 10, 12,
4, 14, 0, 2, 6, 8, 10, 12, 0, 4, 14, 2, 6, 8, 10, 12,
2, 4, 14, 0, 6, 8, 10, 12, 0, 2, 4, 14, 6, 8, 10, 12,
6, 14, 0, 2, 4, 8, 10, 12, 0, 6, 14, 2, 4, 8, 10, 12,
2, 6, 14, 0, 4, 8, 10, 12, 0, 2, 6, 14, 4, 8, 10, 12,
4, 6, 14, 0, 2, 8, 10, 12, 0, 4, 6, 14, 2, 8, 10, 12,
2, 4, 6, 14, 0, 8, 10, 12, 0, 2, 4, 6, 14, 8, 10, 12,
8, 14, 0, 2, 4, 6, 10, 12, 0, 8, 14, 2, 4, 6, 10, 12,
2, 8, 14, 0, 4, 6, 10, 12, 0, 2, 8, 14, 4, 6, 10, 12,
4, 8, 14, 0, 2, 6, 10, 12, 0, 4, 8, 14, 2, 6, 10, 12,
2, 4, 8, 14, 0, 6, 10, 12, 0, 2, 4, 8, 14, 6, 10, 12,
6, 8, 14, 0, 2, 4, 10, 12, 0, 6, 8, 14, 2, 4, 10, 12,
2, 6, 8, 14, 0, 4, 10, 12, 0, 2, 6, 8, 14, 4, 10, 12,
4, 6, 8, 14, 0, 2, 10, 12, 0, 4, 6, 8, 14, 2, 10, 12,
2, 4, 6, 8, 14, 0, 10, 12, 0, 2, 4, 6, 8, 14, 10, 12,
10, 14, 0, 2, 4, 6, 8, 12, 0, 10, 14, 2, 4, 6, 8, 12,
2, 10, 14, 0, 4, 6, 8, 12, 0, 2, 10, 14, 4, 6, 8, 12,
4, 10, 14, 0, 2, 6, 8, 12, 0, 4, 10, 14, 2, 6, 8, 12,
2, 4, 10, 14, 0, 6, 8, 12, 0, 2, 4, 10, 14, 6, 8, 12,
6, 10, 14, 0, 2, 4, 8, 12, 0, 6, 10, 14, 2, 4, 8, 12,
2, 6, 10, 14, 0, 4, 8, 12, 0, 2, 6, 10, 14, 4, 8, 12,
4, 6, 10, 14, 0, 2, 8, 12, 0, 4, 6, 10, 14, 2, 8, 12,
2, 4, 6, 10, 14, 0, 8, 12, 0, 2, 4, 6, 10, 14, 8, 12,
8, 10, 14, 0, 2, 4, 6, 12, 0, 8, 10, 14, 2, 4, 6, 12,
2, 8, 10, 14, 0, 4, 6, 12, 0, 2, 8, 10, 14, 4, 6, 12,
4, 8, 10, 14, 0, 2, 6, 12, 0, 4, 8, 10, 14, 2, 6, 12,
2, 4, 8, 10, 14, 0, 6, 12, 0, 2, 4, 8, 10, 14, 6, 12,
6, 8, 10, 14, 0, 2, 4, 12, 0, 6, 8, 10, 14, 2, 4, 12,
2, 6, 8, 10, 14, 0, 4, 12, 0, 2, 6, 8, 10, 14, 4, 12,
4, 6, 8, 10, 14, 0, 2, 12, 0, 4, 6, 8, 10, 14, 2, 12,
2, 4, 6, 8, 10, 14, 0, 12, 0, 2, 4, 6, 8, 10, 14, 12,
12, 14, 0, 2, 4, 6, 8, 10, 0, 12, 14, 2, 4, 6, 8, 10,
2, 12, 14, 0, 4, 6, 8, 10, 0, 2, 12, 14, 4, 6, 8, 10,
4, 12, 14, 0, 2, 6, 8, 10, 0, 4, 12, 14, 2, 6, 8, 10,
2, 4, 12, 14, 0, 6, 8, 10, 0, 2, 4, 12, 14, 6, 8, 10,
6, 12, 14, 0, 2, 4, 8, 10, 0, 6, 12, 14, 2, 4, 8, 10,
2, 6, 12, 14, 0, 4, 8, 10, 0, 2, 6, 12, 14, 4, 8, 10,
4, 6, 12, 14, 0, 2, 8, 10, 0, 4, 6, 12, 14, 2, 8, 10,
2, 4, 6, 12, 14, 0, 8, 10, 0, 2, 4, 6, 12, 14, 8, 10,
8, 12, 14, 0, 2, 4, 6, 10, 0, 8, 12, 14, 2, 4, 6, 10,
2, 8, 12, 14, 0, 4, 6, 10, 0, 2, 8, 12, 14, 4, 6, 10,
4, 8, 12, 14, 0, 2, 6, 10, 0, 4, 8, 12, 14, 2, 6, 10,
2, 4, 8, 12, 14, 0, 6, 10, 0, 2, 4, 8, 12, 14, 6, 10,
6, 8, 12, 14, 0, 2, 4, 10, 0, 6, 8, 12, 14, 2, 4, 10,
2, 6, 8, 12, 14, 0, 4, 10, 0, 2, 6, 8, 12, 14, 4, 10,
4, 6, 8, 12, 14, 0, 2, 10, 0, 4, 6, 8, 12, 14, 2, 10,
2, 4, 6, 8, 12, 14, 0, 10, 0, 2, 4, 6, 8, 12, 14, 10,
10, 12, 14, 0, 2, 4, 6, 8, 0, 10, 12, 14, 2, 4, 6, 8,
2, 10, 12, 14, 0, 4, 6, 8, 0, 2, 10, 12, 14, 4, 6, 8,
4, 10, 12, 14, 0, 2, 6, 8, 0, 4, 10, 12, 14, 2, 6, 8,
2, 4, 10, 12, 14, 0, 6, 8, 0, 2, 4, 10, 12, 14, 6, 8,
6, 10, 12, 14, 0, 2, 4, 8, 0, 6, 10, 12, 14, 2, 4, 8,
2, 6, 10, 12, 14, 0, 4, 8, 0, 2, 6, 10, 12, 14, 4, 8,
4, 6, 10, 12, 14, 0, 2, 8, 0, 4, 6, 10, 12, 14, 2, 8,
2, 4, 6, 10, 12, 14, 0, 8, 0, 2, 4, 6, 10, 12, 14, 8,
8, 10, 12, 14, 0, 2, 4, 6, 0, 8, 10, 12, 14, 2, 4, 6,
2, 8, 10, 12, 14, 0, 4, 6, 0, 2, 8, 10, 12, 14, 4, 6,
4, 8, 10, 12, 14, 0, 2, 6, 0, 4, 8, 10, 12, 14, 2, 6,
2, 4, 8, 10, 12, 14, 0, 6, 0, 2, 4, 8, 10, 12, 14, 6,
6, 8, 10, 12, 14, 0, 2, 4, 0, 6, 8, 10, 12, 14, 2, 4,
2, 6, 8, 10, 12, 14, 0, 4, 0, 2, 6, 8, 10, 12, 14, 4,
4, 6, 8, 10, 12, 14, 0, 2, 0, 4, 6, 8, 10, 12, 14, 2,
2, 4, 6, 8, 10, 12, 14, 0, 0, 2, 4, 6, 8, 10, 12, 14};
const VFromD<decltype(d8t)> byte_idx{Load(d8, table + mask_bits * 8).raw};
const VFromD<decltype(du)> pairs = ZipLower(byte_idx, byte_idx);
return BitCast(d, pairs + Set(du, 0x0100));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromNotBits128(D d, uint64_t mask_bits) {
do { } while (0);
const Rebind<uint8_t, decltype(d)> d8;
const Twice<decltype(d8)> d8t;
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint8_t table[2048] = {
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0,
0, 4, 6, 8, 10, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2,
0, 2, 6, 8, 10, 12, 14, 4, 2, 6, 8, 10, 12, 14, 0, 4,
0, 6, 8, 10, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4,
0, 2, 4, 8, 10, 12, 14, 6, 2, 4, 8, 10, 12, 14, 0, 6,
0, 4, 8, 10, 12, 14, 2, 6, 4, 8, 10, 12, 14, 0, 2, 6,
0, 2, 8, 10, 12, 14, 4, 6, 2, 8, 10, 12, 14, 0, 4, 6,
0, 8, 10, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6,
0, 2, 4, 6, 10, 12, 14, 8, 2, 4, 6, 10, 12, 14, 0, 8,
0, 4, 6, 10, 12, 14, 2, 8, 4, 6, 10, 12, 14, 0, 2, 8,
0, 2, 6, 10, 12, 14, 4, 8, 2, 6, 10, 12, 14, 0, 4, 8,
0, 6, 10, 12, 14, 2, 4, 8, 6, 10, 12, 14, 0, 2, 4, 8,
0, 2, 4, 10, 12, 14, 6, 8, 2, 4, 10, 12, 14, 0, 6, 8,
0, 4, 10, 12, 14, 2, 6, 8, 4, 10, 12, 14, 0, 2, 6, 8,
0, 2, 10, 12, 14, 4, 6, 8, 2, 10, 12, 14, 0, 4, 6, 8,
0, 10, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8,
0, 2, 4, 6, 8, 12, 14, 10, 2, 4, 6, 8, 12, 14, 0, 10,
0, 4, 6, 8, 12, 14, 2, 10, 4, 6, 8, 12, 14, 0, 2, 10,
0, 2, 6, 8, 12, 14, 4, 10, 2, 6, 8, 12, 14, 0, 4, 10,
0, 6, 8, 12, 14, 2, 4, 10, 6, 8, 12, 14, 0, 2, 4, 10,
0, 2, 4, 8, 12, 14, 6, 10, 2, 4, 8, 12, 14, 0, 6, 10,
0, 4, 8, 12, 14, 2, 6, 10, 4, 8, 12, 14, 0, 2, 6, 10,
0, 2, 8, 12, 14, 4, 6, 10, 2, 8, 12, 14, 0, 4, 6, 10,
0, 8, 12, 14, 2, 4, 6, 10, 8, 12, 14, 0, 2, 4, 6, 10,
0, 2, 4, 6, 12, 14, 8, 10, 2, 4, 6, 12, 14, 0, 8, 10,
0, 4, 6, 12, 14, 2, 8, 10, 4, 6, 12, 14, 0, 2, 8, 10,
0, 2, 6, 12, 14, 4, 8, 10, 2, 6, 12, 14, 0, 4, 8, 10,
0, 6, 12, 14, 2, 4, 8, 10, 6, 12, 14, 0, 2, 4, 8, 10,
0, 2, 4, 12, 14, 6, 8, 10, 2, 4, 12, 14, 0, 6, 8, 10,
0, 4, 12, 14, 2, 6, 8, 10, 4, 12, 14, 0, 2, 6, 8, 10,
0, 2, 12, 14, 4, 6, 8, 10, 2, 12, 14, 0, 4, 6, 8, 10,
0, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10,
0, 2, 4, 6, 8, 10, 14, 12, 2, 4, 6, 8, 10, 14, 0, 12,
0, 4, 6, 8, 10, 14, 2, 12, 4, 6, 8, 10, 14, 0, 2, 12,
0, 2, 6, 8, 10, 14, 4, 12, 2, 6, 8, 10, 14, 0, 4, 12,
0, 6, 8, 10, 14, 2, 4, 12, 6, 8, 10, 14, 0, 2, 4, 12,
0, 2, 4, 8, 10, 14, 6, 12, 2, 4, 8, 10, 14, 0, 6, 12,
0, 4, 8, 10, 14, 2, 6, 12, 4, 8, 10, 14, 0, 2, 6, 12,
0, 2, 8, 10, 14, 4, 6, 12, 2, 8, 10, 14, 0, 4, 6, 12,
0, 8, 10, 14, 2, 4, 6, 12, 8, 10, 14, 0, 2, 4, 6, 12,
0, 2, 4, 6, 10, 14, 8, 12, 2, 4, 6, 10, 14, 0, 8, 12,
0, 4, 6, 10, 14, 2, 8, 12, 4, 6, 10, 14, 0, 2, 8, 12,
0, 2, 6, 10, 14, 4, 8, 12, 2, 6, 10, 14, 0, 4, 8, 12,
0, 6, 10, 14, 2, 4, 8, 12, 6, 10, 14, 0, 2, 4, 8, 12,
0, 2, 4, 10, 14, 6, 8, 12, 2, 4, 10, 14, 0, 6, 8, 12,
0, 4, 10, 14, 2, 6, 8, 12, 4, 10, 14, 0, 2, 6, 8, 12,
0, 2, 10, 14, 4, 6, 8, 12, 2, 10, 14, 0, 4, 6, 8, 12,
0, 10, 14, 2, 4, 6, 8, 12, 10, 14, 0, 2, 4, 6, 8, 12,
0, 2, 4, 6, 8, 14, 10, 12, 2, 4, 6, 8, 14, 0, 10, 12,
0, 4, 6, 8, 14, 2, 10, 12, 4, 6, 8, 14, 0, 2, 10, 12,
0, 2, 6, 8, 14, 4, 10, 12, 2, 6, 8, 14, 0, 4, 10, 12,
0, 6, 8, 14, 2, 4, 10, 12, 6, 8, 14, 0, 2, 4, 10, 12,
0, 2, 4, 8, 14, 6, 10, 12, 2, 4, 8, 14, 0, 6, 10, 12,
0, 4, 8, 14, 2, 6, 10, 12, 4, 8, 14, 0, 2, 6, 10, 12,
0, 2, 8, 14, 4, 6, 10, 12, 2, 8, 14, 0, 4, 6, 10, 12,
0, 8, 14, 2, 4, 6, 10, 12, 8, 14, 0, 2, 4, 6, 10, 12,
0, 2, 4, 6, 14, 8, 10, 12, 2, 4, 6, 14, 0, 8, 10, 12,
0, 4, 6, 14, 2, 8, 10, 12, 4, 6, 14, 0, 2, 8, 10, 12,
0, 2, 6, 14, 4, 8, 10, 12, 2, 6, 14, 0, 4, 8, 10, 12,
0, 6, 14, 2, 4, 8, 10, 12, 6, 14, 0, 2, 4, 8, 10, 12,
0, 2, 4, 14, 6, 8, 10, 12, 2, 4, 14, 0, 6, 8, 10, 12,
0, 4, 14, 2, 6, 8, 10, 12, 4, 14, 0, 2, 6, 8, 10, 12,
0, 2, 14, 4, 6, 8, 10, 12, 2, 14, 0, 4, 6, 8, 10, 12,
0, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12,
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 6, 8, 10, 12, 0, 14,
0, 4, 6, 8, 10, 12, 2, 14, 4, 6, 8, 10, 12, 0, 2, 14,
0, 2, 6, 8, 10, 12, 4, 14, 2, 6, 8, 10, 12, 0, 4, 14,
0, 6, 8, 10, 12, 2, 4, 14, 6, 8, 10, 12, 0, 2, 4, 14,
0, 2, 4, 8, 10, 12, 6, 14, 2, 4, 8, 10, 12, 0, 6, 14,
0, 4, 8, 10, 12, 2, 6, 14, 4, 8, 10, 12, 0, 2, 6, 14,
0, 2, 8, 10, 12, 4, 6, 14, 2, 8, 10, 12, 0, 4, 6, 14,
0, 8, 10, 12, 2, 4, 6, 14, 8, 10, 12, 0, 2, 4, 6, 14,
0, 2, 4, 6, 10, 12, 8, 14, 2, 4, 6, 10, 12, 0, 8, 14,
0, 4, 6, 10, 12, 2, 8, 14, 4, 6, 10, 12, 0, 2, 8, 14,
0, 2, 6, 10, 12, 4, 8, 14, 2, 6, 10, 12, 0, 4, 8, 14,
0, 6, 10, 12, 2, 4, 8, 14, 6, 10, 12, 0, 2, 4, 8, 14,
0, 2, 4, 10, 12, 6, 8, 14, 2, 4, 10, 12, 0, 6, 8, 14,
0, 4, 10, 12, 2, 6, 8, 14, 4, 10, 12, 0, 2, 6, 8, 14,
0, 2, 10, 12, 4, 6, 8, 14, 2, 10, 12, 0, 4, 6, 8, 14,
0, 10, 12, 2, 4, 6, 8, 14, 10, 12, 0, 2, 4, 6, 8, 14,
0, 2, 4, 6, 8, 12, 10, 14, 2, 4, 6, 8, 12, 0, 10, 14,
0, 4, 6, 8, 12, 2, 10, 14, 4, 6, 8, 12, 0, 2, 10, 14,
0, 2, 6, 8, 12, 4, 10, 14, 2, 6, 8, 12, 0, 4, 10, 14,
0, 6, 8, 12, 2, 4, 10, 14, 6, 8, 12, 0, 2, 4, 10, 14,
0, 2, 4, 8, 12, 6, 10, 14, 2, 4, 8, 12, 0, 6, 10, 14,
0, 4, 8, 12, 2, 6, 10, 14, 4, 8, 12, 0, 2, 6, 10, 14,
0, 2, 8, 12, 4, 6, 10, 14, 2, 8, 12, 0, 4, 6, 10, 14,
0, 8, 12, 2, 4, 6, 10, 14, 8, 12, 0, 2, 4, 6, 10, 14,
0, 2, 4, 6, 12, 8, 10, 14, 2, 4, 6, 12, 0, 8, 10, 14,
0, 4, 6, 12, 2, 8, 10, 14, 4, 6, 12, 0, 2, 8, 10, 14,
0, 2, 6, 12, 4, 8, 10, 14, 2, 6, 12, 0, 4, 8, 10, 14,
0, 6, 12, 2, 4, 8, 10, 14, 6, 12, 0, 2, 4, 8, 10, 14,
0, 2, 4, 12, 6, 8, 10, 14, 2, 4, 12, 0, 6, 8, 10, 14,
0, 4, 12, 2, 6, 8, 10, 14, 4, 12, 0, 2, 6, 8, 10, 14,
0, 2, 12, 4, 6, 8, 10, 14, 2, 12, 0, 4, 6, 8, 10, 14,
0, 12, 2, 4, 6, 8, 10, 14, 12, 0, 2, 4, 6, 8, 10, 14,
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 6, 8, 10, 0, 12, 14,
0, 4, 6, 8, 10, 2, 12, 14, 4, 6, 8, 10, 0, 2, 12, 14,
0, 2, 6, 8, 10, 4, 12, 14, 2, 6, 8, 10, 0, 4, 12, 14,
0, 6, 8, 10, 2, 4, 12, 14, 6, 8, 10, 0, 2, 4, 12, 14,
0, 2, 4, 8, 10, 6, 12, 14, 2, 4, 8, 10, 0, 6, 12, 14,
0, 4, 8, 10, 2, 6, 12, 14, 4, 8, 10, 0, 2, 6, 12, 14,
0, 2, 8, 10, 4, 6, 12, 14, 2, 8, 10, 0, 4, 6, 12, 14,
0, 8, 10, 2, 4, 6, 12, 14, 8, 10, 0, 2, 4, 6, 12, 14,
0, 2, 4, 6, 10, 8, 12, 14, 2, 4, 6, 10, 0, 8, 12, 14,
0, 4, 6, 10, 2, 8, 12, 14, 4, 6, 10, 0, 2, 8, 12, 14,
0, 2, 6, 10, 4, 8, 12, 14, 2, 6, 10, 0, 4, 8, 12, 14,
0, 6, 10, 2, 4, 8, 12, 14, 6, 10, 0, 2, 4, 8, 12, 14,
0, 2, 4, 10, 6, 8, 12, 14, 2, 4, 10, 0, 6, 8, 12, 14,
0, 4, 10, 2, 6, 8, 12, 14, 4, 10, 0, 2, 6, 8, 12, 14,
0, 2, 10, 4, 6, 8, 12, 14, 2, 10, 0, 4, 6, 8, 12, 14,
0, 10, 2, 4, 6, 8, 12, 14, 10, 0, 2, 4, 6, 8, 12, 14,
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 6, 8, 0, 10, 12, 14,
0, 4, 6, 8, 2, 10, 12, 14, 4, 6, 8, 0, 2, 10, 12, 14,
0, 2, 6, 8, 4, 10, 12, 14, 2, 6, 8, 0, 4, 10, 12, 14,
0, 6, 8, 2, 4, 10, 12, 14, 6, 8, 0, 2, 4, 10, 12, 14,
0, 2, 4, 8, 6, 10, 12, 14, 2, 4, 8, 0, 6, 10, 12, 14,
0, 4, 8, 2, 6, 10, 12, 14, 4, 8, 0, 2, 6, 10, 12, 14,
0, 2, 8, 4, 6, 10, 12, 14, 2, 8, 0, 4, 6, 10, 12, 14,
0, 8, 2, 4, 6, 10, 12, 14, 8, 0, 2, 4, 6, 10, 12, 14,
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 6, 0, 8, 10, 12, 14,
0, 4, 6, 2, 8, 10, 12, 14, 4, 6, 0, 2, 8, 10, 12, 14,
0, 2, 6, 4, 8, 10, 12, 14, 2, 6, 0, 4, 8, 10, 12, 14,
0, 6, 2, 4, 8, 10, 12, 14, 6, 0, 2, 4, 8, 10, 12, 14,
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 0, 6, 8, 10, 12, 14,
0, 4, 2, 6, 8, 10, 12, 14, 4, 0, 2, 6, 8, 10, 12, 14,
0, 2, 4, 6, 8, 10, 12, 14, 2, 0, 4, 6, 8, 10, 12, 14,
0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14};
const VFromD<decltype(d8t)> byte_idx{Load(d8, table + mask_bits * 8).raw};
const VFromD<decltype(du)> pairs = ZipLower(byte_idx, byte_idx);
return BitCast(d, pairs + Set(du, 0x0100));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromBits128(D d, uint64_t mask_bits) {
do { } while (0);
alignas(16) static constexpr uint8_t u8_indices[256] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
8, 9, 10, 11, 0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15,
0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15,
4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 12, 13, 14, 15,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7, 8, 9, 10, 11,
4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 8, 9, 10, 11,
0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 8, 9, 10, 11,
8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15, 4, 5, 6, 7,
4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, Load(d8, u8_indices + 16 * mask_bits));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromNotBits128(D d, uint64_t mask_bits) {
do { } while (0);
alignas(16) static constexpr uint8_t u8_indices[256] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 5,
6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3,
8, 9, 10, 11, 12, 13, 14, 15, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15, 0, 1,
2, 3, 8, 9, 10, 11, 0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7,
8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
10, 11, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 12, 13, 14, 15, 0, 1,
2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15, 8, 9, 10, 11,
0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5,
6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 5, 6, 7, 0, 1, 2, 3,
8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
12, 13, 14, 15};
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, Load(d8, u8_indices + 16 * mask_bits));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromBits128(D d, uint64_t mask_bits) {
do { } while (0);
alignas(16) static constexpr uint8_t u8_indices[64] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, Load(d8, u8_indices + 16 * mask_bits));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromNotBits128(D d, uint64_t mask_bits) {
do { } while (0);
alignas(16) static constexpr uint8_t u8_indices[64] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, Load(d8, u8_indices + 16 * mask_bits));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> CompressBits(Vec128<T, N> v, uint64_t mask_bits) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
do { } while (0);
const auto indices = BitCast(du, detail::IndicesFromBits128(d, mask_bits));
return BitCast(d, TableLookupBytes(BitCast(du, v), indices));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> CompressNotBits(Vec128<T, N> v, uint64_t mask_bits) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
do { } while (0);
const auto indices = BitCast(du, detail::IndicesFromNotBits128(d, mask_bits));
return BitCast(d, TableLookupBytes(BitCast(du, v), indices));
}
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> Compress(Vec128<T, 1> v, Mask128<T, 1> ) {
return v;
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Compress(Vec128<T> v, Mask128<T> mask) {
const DFromV<decltype(v)> d;
const Vec128<T> m = VecFromMask(d, mask);
const Vec128<T> maskL = DupEven(m);
const Vec128<T> maskH = DupOdd(m);
const Vec128<T> swap = AndNot(maskL, maskH);
return IfVecThenElse(swap, Shuffle01(v), v);
}
template <typename T, size_t N, hwy::EnableIf<((size_t{1} << sizeof(T)) & ((1 << 2) | (1 << 4))) != 0>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Compress(Vec128<T, N> v, Mask128<T, N> mask) {
return detail::CompressBits(v, detail::BitsFromMask(mask));
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> CompressNot(Vec128<T, 1> v, Mask128<T, 1> ) {
return v;
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> CompressNot(Vec128<T> v, Mask128<T> mask) {
const DFromV<decltype(v)> d;
const Vec128<T> m = VecFromMask(d, mask);
const Vec128<T> maskL = DupEven(m);
const Vec128<T> maskH = DupOdd(m);
const Vec128<T> swap = AndNot(maskH, maskL);
return IfVecThenElse(swap, Shuffle01(v), v);
}
template <typename T, size_t N, hwy::EnableIf<((size_t{1} << sizeof(T)) & ((1 << 2) | (1 << 4))) != 0>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> CompressNot(Vec128<T, N> v, Mask128<T, N> mask) {
if (N < 16 / sizeof(T)) {
return detail::CompressBits(v, detail::BitsFromMask(Not(mask)));
}
return detail::CompressNotBits(v, detail::BitsFromMask(mask));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t> CompressBlocksNot(Vec128<uint64_t> v,
Mask128<uint64_t> ) {
return v;
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> CompressBits(Vec128<T, N> v,
const uint8_t* __restrict__ bits) {
uint64_t mask_bits = 0;
constexpr size_t kNumBytes = (N + 7) / 8;
CopyBytes<kNumBytes>(bits, &mask_bits);
if (N < 8) {
mask_bits &= (1ull << N) - 1;
}
return detail::CompressBits(v, mask_bits);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressStore(VFromD<D> v, MFromD<D> m, D d,
TFromD<D>* __restrict__ unaligned) {
const RebindToUnsigned<decltype(d)> du;
const uint64_t mask_bits = detail::BitsFromMask(m);
do { } while (0);
const size_t count = PopCount(mask_bits);
const auto indices = BitCast(du, detail::IndicesFromBits128(d, mask_bits));
const auto compressed = BitCast(d, TableLookupBytes(BitCast(du, v), indices));
StoreU(compressed, d, unaligned);
detail::MaybeUnpoison(unaligned, count);
return count;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressBlendedStore(VFromD<D> v, MFromD<D> m, D d,
TFromD<D>* __restrict__ unaligned) {
const RebindToUnsigned<decltype(d)> du;
const uint64_t mask_bits = detail::BitsFromMask(m);
do { } while (0);
const size_t count = PopCount(mask_bits);
const auto indices = BitCast(du, detail::IndicesFromBits128(d, mask_bits));
const auto compressed = BitCast(d, TableLookupBytes(BitCast(du, v), indices));
BlendedStore(compressed, FirstN(d, count), d, unaligned);
detail::MaybeUnpoison(unaligned, count);
return count;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressBitsStore(VFromD<D> v, const uint8_t* __restrict__ bits,
D d, TFromD<D>* __restrict__ unaligned) {
const RebindToUnsigned<decltype(d)> du;
uint64_t mask_bits = 0;
constexpr size_t kN = MaxLanes(d);
constexpr size_t kNumBytes = (kN + 7) / 8;
CopyBytes<kNumBytes>(bits, &mask_bits);
if (kN < 8) {
mask_bits &= (1ull << kN) - 1;
}
const size_t count = PopCount(mask_bits);
const auto indices = BitCast(du, detail::IndicesFromBits128(d, mask_bits));
const auto compressed = BitCast(d, TableLookupBytes(BitCast(du, v), indices));
StoreU(compressed, d, unaligned);
detail::MaybeUnpoison(unaligned, count);
return count;
}
template <class T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, 1> SetAtOrAfterFirst(Mask128<T, 1> mask) {
return mask;
}
template <class T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, 2> SetAtOrAfterFirst(Mask128<T, 2> mask) {
const FixedTag<T, 2> d;
const auto vmask = VecFromMask(d, mask);
return MaskFromVec(Or(vmask, InterleaveLower(vmask, vmask)));
}
template <class T, size_t N, hwy::EnableIf<(N > 2)>* = nullptr, hwy::EnableIf<N * sizeof(T) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> SetAtOrAfterFirst(Mask128<T, N> mask) {
const Simd<T, N, 0> d;
const auto vmask = VecFromMask(d, mask);
const auto neg_vmask =
ResizeBitCast(d, Neg(ResizeBitCast(Full64<int64_t>(), vmask)));
return MaskFromVec(Or(vmask, neg_vmask));
}
template <class T, hwy::EnableIf<sizeof(T) != (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T> SetAtOrAfterFirst(Mask128<T> mask) {
const Full128<T> d;
const Repartition<int64_t, decltype(d)> di64;
const Repartition<float, decltype(d)> df32;
const Repartition<int32_t, decltype(d)> di32;
using VF = VFromD<decltype(df32)>;
auto vmask = BitCast(di64, VecFromMask(d, mask));
vmask = Or(vmask, Neg(vmask));
const auto vmask2 = BroadcastSignBit(
BitCast(di32, VF{_mm_shuffle_ps(Zero(df32).raw, BitCast(df32, vmask).raw,
(((
1
) << 6) | ((
1
) << 4) | ((
0
) << 2) | (
0
))
)}));
return MaskFromVec(BitCast(d, Or(vmask, BitCast(di64, vmask2))));
}
template <class T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> SetBeforeFirst(Mask128<T, N> mask) {
return Not(SetAtOrAfterFirst(mask));
}
template <class T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, 1> SetOnlyFirst(Mask128<T, 1> mask) {
return mask;
}
template <class T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, 2> SetOnlyFirst(Mask128<T, 2> mask) {
const FixedTag<T, 2> d;
const RebindToSigned<decltype(d)> di;
const auto vmask = BitCast(di, VecFromMask(d, mask));
const auto zero = Zero(di);
const auto vmask2 = VecFromMask(di, InterleaveLower(zero, vmask) == zero);
return MaskFromVec(BitCast(d, And(vmask, vmask2)));
}
template <class T, size_t N, hwy::EnableIf<(N > 2)>* = nullptr, hwy::EnableIf<N * sizeof(T) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> SetOnlyFirst(Mask128<T, N> mask) {
const Simd<T, N, 0> d;
const RebindToSigned<decltype(d)> di;
const auto vmask = ResizeBitCast(Full64<int64_t>(), VecFromMask(d, mask));
const auto only_first_vmask =
BitCast(d, Neg(ResizeBitCast(di, And(vmask, Neg(vmask)))));
return MaskFromVec(only_first_vmask);
}
template <class T, hwy::EnableIf<sizeof(T) != (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T> SetOnlyFirst(Mask128<T> mask) {
const Full128<T> d;
const RebindToSigned<decltype(d)> di;
const Repartition<int64_t, decltype(d)> di64;
const auto zero = Zero(di64);
const auto vmask = BitCast(di64, VecFromMask(d, mask));
const auto vmask2 = VecFromMask(di64, InterleaveLower(zero, vmask) == zero);
const auto only_first_vmask = Neg(BitCast(di, And(vmask, Neg(vmask))));
return MaskFromVec(BitCast(d, And(only_first_vmask, BitCast(di, vmask2))));
}
template <class T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, 1> SetAtOrBeforeFirst(Mask128<T, 1> ) {
const FixedTag<T, 1> d;
const RebindToSigned<decltype(d)> di;
using TI = MakeSigned<T>;
return RebindMask(d, MaskFromVec(Set(di, TI(-1))));
}
template <class T, size_t N, hwy::EnableIf<(N > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> SetAtOrBeforeFirst(Mask128<T, N> mask) {
const Simd<T, N, 0> d;
return SetBeforeFirst(MaskFromVec(ShiftLeftLanes<1>(VecFromMask(d, mask))));
}
namespace detail {
template <typename T>
inline __attribute__((always_inline)) Vec128<T, 1> SumOfLanes(Vec128<T, 1> v) {
return v;
}
template <typename T>
inline __attribute__((always_inline)) T ReduceSum(Vec128<T, 1> v) {
return GetLane(v);
}
template <typename T>
inline __attribute__((always_inline)) Vec128<T, 1> MinOfLanes(Vec128<T, 1> v) {
return v;
}
template <typename T>
inline __attribute__((always_inline)) Vec128<T, 1> MaxOfLanes(Vec128<T, 1> v) {
return v;
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, 2> SumOfLanes(Vec128<T, 2> v10) {
return v10 + Shuffle2301(v10);
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) T ReduceSum(Vec128<T, 2> v10) {
return GetLane(SumOfLanes(v10));
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, 2> MinOfLanes(Vec128<T, 2> v10) {
return Min(v10, Shuffle2301(v10));
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, 2> MaxOfLanes(Vec128<T, 2> v10) {
return Max(v10, Shuffle2301(v10));
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> SumOfLanes(Vec128<T> v3210) {
const Vec128<T> v1032 = Shuffle1032(v3210);
const Vec128<T> v31_20_31_20 = v3210 + v1032;
const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20);
return v20_31_20_31 + v31_20_31_20;
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) T ReduceSum(Vec128<T> v3210) {
return GetLane(SumOfLanes(v3210));
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> MinOfLanes(Vec128<T> v3210) {
const Vec128<T> v1032 = Shuffle1032(v3210);
const Vec128<T> v31_20_31_20 = Min(v3210, v1032);
const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20);
return Min(v20_31_20_31, v31_20_31_20);
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> MaxOfLanes(Vec128<T> v3210) {
const Vec128<T> v1032 = Shuffle1032(v3210);
const Vec128<T> v31_20_31_20 = Max(v3210, v1032);
const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20);
return Max(v20_31_20_31, v31_20_31_20);
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> SumOfLanes(Vec128<T> v10) {
const Vec128<T> v01 = Shuffle01(v10);
return v10 + v01;
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) T ReduceSum(Vec128<T> v10) {
return GetLane(SumOfLanes(v10));
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> MinOfLanes(Vec128<T> v10) {
const Vec128<T> v01 = Shuffle01(v10);
return Min(v10, v01);
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> MaxOfLanes(Vec128<T> v10) {
const Vec128<T> v01 = Shuffle01(v10);
return Max(v10, v01);
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) uint16_t ReduceSum(Vec128<uint16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto sum = ReduceSum(even + odd);
return static_cast<uint16_t>(sum);
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<uint16_t, N> SumOfLanes(Vec128<uint16_t, N> v) {
const DFromV<decltype(v)> d;
return Set(d, ReduceSum(v));
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) int16_t ReduceSum(Vec128<int16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = ShiftRight<16>(ShiftLeft<16>(BitCast(d32, v)));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto sum = ReduceSum(even + odd);
return static_cast<int16_t>(sum);
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int16_t, N> SumOfLanes(Vec128<int16_t, N> v) {
const DFromV<decltype(v)> d;
return Set(d, ReduceSum(v));
}
inline __attribute__((always_inline)) uint8_t ReduceSum(Vec64<uint8_t> v) {
return static_cast<uint8_t>(GetLane(SumsOf8(v)) & 0xFF);
}
inline __attribute__((always_inline)) Vec64<uint8_t> SumOfLanes(Vec64<uint8_t> v) {
const Full64<uint8_t> d;
return Set(d, ReduceSum(v));
}
inline __attribute__((always_inline)) uint8_t ReduceSum(Vec128<uint8_t> v) {
uint64_t sums = ReduceSum(SumsOf8(v));
return static_cast<uint8_t>(sums & 0xFF);
}
inline __attribute__((always_inline)) Vec128<uint8_t> SumOfLanes(Vec128<uint8_t> v) {
const DFromV<decltype(v)> d;
return Set(d, ReduceSum(v));
}
template <size_t N, hwy::EnableIf<(N * sizeof(int8_t) > 4)>* = nullptr>
inline __attribute__((always_inline)) int8_t ReduceSum(const Vec128<int8_t, N> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const auto is_neg = v < Zero(d);
const auto positive = SumsOf8(BitCast(du, IfThenZeroElse(is_neg, v)));
const auto negative = SumsOf8(BitCast(du, IfThenElseZero(is_neg, Abs(v))));
return static_cast<int8_t>(ReduceSum(positive - negative) & 0xFF);
}
template <size_t N, hwy::EnableIf<(N * sizeof(int8_t) > 4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int8_t, N> SumOfLanes(const Vec128<int8_t, N> v) {
const DFromV<decltype(v)> d;
return Set(d, ReduceSum(v));
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint8_t) > 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> MaxOfLanes(Vec128<uint8_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d16;
const RepartitionToWide<decltype(d16)> d32;
Vec128<uint8_t, N> vm = Max(v, Reverse2(d, v));
vm = Max(vm, BitCast(d, Reverse2(d16, BitCast(d16, vm))));
vm = Max(vm, BitCast(d, Reverse2(d32, BitCast(d32, vm))));
if (N > 8) {
const RepartitionToWide<decltype(d32)> d64;
vm = Max(vm, BitCast(d, Reverse2(d64, BitCast(d64, vm))));
}
return vm;
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint8_t) > 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> MinOfLanes(Vec128<uint8_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d16;
const RepartitionToWide<decltype(d16)> d32;
Vec128<uint8_t, N> vm = Min(v, Reverse2(d, v));
vm = Min(vm, BitCast(d, Reverse2(d16, BitCast(d16, vm))));
vm = Min(vm, BitCast(d, Reverse2(d32, BitCast(d32, vm))));
if (N > 8) {
const RepartitionToWide<decltype(d32)> d64;
vm = Min(vm, BitCast(d, Reverse2(d64, BitCast(d64, vm))));
}
return vm;
}
template <size_t N, hwy::EnableIf<(N * sizeof(int8_t) > 4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int8_t, N> MinOfLanes(Vec128<int8_t, N> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const auto mask = SignBit(du);
const auto vu = Xor(BitCast(du, v), mask);
return BitCast(d, Xor(MinOfLanes(vu), mask));
}
template <size_t N, hwy::EnableIf<(N * sizeof(int8_t) > 4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int8_t, N> MaxOfLanes(Vec128<int8_t, N> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const auto mask = SignBit(du);
const auto vu = Xor(BitCast(du, v), mask);
return BitCast(d, Xor(MaxOfLanes(vu), mask));
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<uint16_t, N> MinOfLanes(Vec128<uint16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto min = MinOfLanes(Min(even, odd));
return OddEven(BitCast(d, ShiftLeft<16>(min)), BitCast(d, min));
}
template <size_t N, hwy::EnableIf<(N * sizeof(int16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int16_t, N> MinOfLanes(Vec128<int16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = ShiftRight<16>(ShiftLeft<16>(BitCast(d32, v)));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto min = MinOfLanes(Min(even, odd));
return OddEven(BitCast(d, ShiftLeft<16>(min)), BitCast(d, min));
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<uint16_t, N> MaxOfLanes(Vec128<uint16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto min = MaxOfLanes(Max(even, odd));
return OddEven(BitCast(d, ShiftLeft<16>(min)), BitCast(d, min));
}
template <size_t N, hwy::EnableIf<(N * sizeof(int16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int16_t, N> MaxOfLanes(Vec128<int16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = ShiftRight<16>(ShiftLeft<16>(BitCast(d32, v)));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto min = MaxOfLanes(Max(even, odd));
return OddEven(BitCast(d, ShiftLeft<16>(min)), BitCast(d, min));
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SumOfLanes(D , VFromD<D> v) {
return detail::SumOfLanes(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) TFromD<D> ReduceSum(D , VFromD<D> v) {
return detail::ReduceSum(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> MinOfLanes(D , VFromD<D> v) {
return detail::MinOfLanes(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> MaxOfLanes(D , VFromD<D> v) {
return detail::MaxOfLanes(v);
}
namespace detail {
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Lt128Vec(const D d, const V a, const V b) {
static_assert(IsSame<TFromD<D>, uint64_t>(), "D must be u64");
const auto eqHL = Eq(a, b);
const V ltHL = VecFromMask(d, Lt(a, b));
const V ltLX = ShiftLeftLanes<1>(ltHL);
const V vecHx = IfThenElse(eqHL, ltLX, ltHL);
return InterleaveUpper(d, vecHx, vecHx);
}
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Eq128Vec(const D d, const V a, const V b) {
static_assert(IsSame<TFromD<D>, uint64_t>(), "D must be u64");
const auto eqHL = VecFromMask(d, Eq(a, b));
const auto eqLH = Reverse2(d, eqHL);
return And(eqHL, eqLH);
}
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Ne128Vec(const D d, const V a, const V b) {
static_assert(IsSame<TFromD<D>, uint64_t>(), "D must be u64");
const auto neHL = VecFromMask(d, Ne(a, b));
const auto neLH = Reverse2(d, neHL);
return Or(neHL, neLH);
}
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Lt128UpperVec(const D d, const V a, const V b) {
const V ltHL = VecFromMask(d, Lt(a, b));
return InterleaveUpper(d, ltHL, ltHL);
}
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Eq128UpperVec(const D d, const V a, const V b) {
const V eqHL = VecFromMask(d, Eq(a, b));
return InterleaveUpper(d, eqHL, eqHL);
}
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Ne128UpperVec(const D d, const V a, const V b) {
const V neHL = VecFromMask(d, Ne(a, b));
return InterleaveUpper(d, neHL, neHL);
}
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Lt128(D d, const V a, const V b) {
return MaskFromVec(detail::Lt128Vec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Eq128(D d, const V a, const V b) {
return MaskFromVec(detail::Eq128Vec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Ne128(D d, const V a, const V b) {
return MaskFromVec(detail::Ne128Vec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Lt128Upper(D d, const V a, const V b) {
return MaskFromVec(detail::Lt128UpperVec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Eq128Upper(D d, const V a, const V b) {
return MaskFromVec(detail::Eq128UpperVec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Ne128Upper(D d, const V a, const V b) {
return MaskFromVec(detail::Ne128UpperVec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Min128(D d, const V a, const V b) {
return IfVecThenElse(detail::Lt128Vec(d, a, b), a, b);
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Max128(D d, const V a, const V b) {
return IfVecThenElse(detail::Lt128Vec(d, b, a), a, b);
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Min128Upper(D d, const V a, const V b) {
return IfVecThenElse(detail::Lt128UpperVec(d, a, b), a, b);
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Max128Upper(D d, const V a, const V b) {
return IfVecThenElse(detail::Lt128UpperVec(d, b, a), a, b);
}
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC diagnostic pop
#pragma GCC push_options
#pragma GCC target "sse2"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSE2 {
template <class V>
using LaneType = decltype(GetLane(V()));
template <class D>
using Vec = decltype(Zero(D()));
template <class D>
using Mask = decltype(MaskFromVec(Zero(D())));
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Clamp(const V v, const V lo, const V hi) {
return Min(Max(lo, v), hi);
}
template <size_t kLanes, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> CombineShiftRightLanes(D d, VFromD<D> hi, VFromD<D> lo) {
constexpr size_t kBytes = kLanes * sizeof(TFromD<D>);
static_assert(kBytes < 16, "Shift count is per-block");
return CombineShiftRightBytes<kBytes>(d, hi, lo);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec<D> SignBit(D d) {
const RebindToUnsigned<decltype(d)> du;
return BitCast(d, Set(du, SignMask<TFromD<D>>()));
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec<D> NaN(D d) {
const RebindToSigned<D> di;
return BitCast(d, Set(di, LimitsMax<TFromD<decltype(di)>>()));
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec<D> Inf(D d) {
const RebindToUnsigned<D> du;
using T = TFromD<D>;
using TU = TFromD<decltype(du)>;
const TU max_x2 = static_cast<TU>(MaxExponentTimes2<T>());
return BitCast(d, Set(du, max_x2 >> 1));
}
namespace detail {
template <size_t kFromVectSize, size_t kToVectSize, class DTo, class DFrom,
hwy::EnableIf<(kToVectSize <= kFromVectSize)>* = nullptr>
inline __attribute__((always_inline)) VFromD<DTo> ZeroExtendResizeBitCast(
hwy::SizeTag<kFromVectSize> ,
hwy::SizeTag<kToVectSize> , DTo d_to, DFrom ,
VFromD<DFrom> v) {
return ResizeBitCast(d_to, v);
}
template <size_t kFromVectSize, size_t kToVectSize, class DTo, class DFrom,
hwy::EnableIf<(kToVectSize == kFromVectSize * 2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<DTo> ZeroExtendResizeBitCast(
hwy::SizeTag<kFromVectSize> ,
hwy::SizeTag<kToVectSize> , DTo d_to, DFrom d_from,
VFromD<DFrom> v) {
const Twice<decltype(d_from)> dt_from;
return BitCast(d_to, ZeroExtendVector(dt_from, v));
}
template <size_t kFromVectSize, size_t kToVectSize, class DTo, class DFrom,
hwy::EnableIf<(kToVectSize > kFromVectSize * 2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<DTo> ZeroExtendResizeBitCast(
hwy::SizeTag<kFromVectSize> ,
hwy::SizeTag<kToVectSize> , DTo d_to, DFrom ,
VFromD<DFrom> v) {
using TFrom = TFromD<DFrom>;
constexpr size_t kNumOfFromLanes = kFromVectSize / sizeof(TFrom);
const Repartition<TFrom, decltype(d_to)> d_resize_to;
return BitCast(d_to, IfThenElseZero(FirstN(d_resize_to, kNumOfFromLanes),
ResizeBitCast(d_resize_to, v)));
}
}
template <class DTo, class DFrom>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DTo> ZeroExtendResizeBitCast(DTo d_to, DFrom d_from,
VFromD<DFrom> v) {
return detail::ZeroExtendResizeBitCast(hwy::SizeTag<d_from.MaxBytes()>(),
hwy::SizeTag<d_to.MaxBytes()>(), d_to,
d_from, v);
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void SafeFillN(const size_t num, const T value, D d,
T* __restrict__ to) {
(void)d;
for (size_t i = 0; i < num; ++i) {
to[i] = value;
}
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void SafeCopyN(const size_t num, D d, const T* __restrict__ from,
T* __restrict__ to) {
(void)d;
for (size_t i = 0; i < num; ++i) {
to[i] = from[i];
}
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V BitwiseIfThenElse(V mask, V yes, V no) {
return Or(And(mask, yes), AndNot(mask, no));
}
template <class D, hwy::EnableIf<(D::kPrivateLanes > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved2(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1) {
const VFromD<D> A = LoadU(d, unaligned);
const VFromD<D> B = LoadU(d, unaligned + Lanes(d));
v0 = ConcatEven(d, B, A);
v1 = ConcatOdd(d, B, A);
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved2(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1) {
v0 = LoadU(d, unaligned + 0);
v1 = LoadU(d, unaligned + 1);
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) void LoadTransposedBlocks3(D d,
const TFromD<D>* __restrict__ unaligned,
VFromD<D>& A, VFromD<D>& B,
VFromD<D>& C) {
constexpr size_t kN = MaxLanes(d);
A = LoadU(d, unaligned + 0 * kN);
B = LoadU(d, unaligned + 1 * kN);
C = LoadU(d, unaligned + 2 * kN);
}
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (16)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
const RebindToUnsigned<decltype(d)> du;
using V = VFromD<D>;
V A;
V B;
V C;
detail::LoadTransposedBlocks3(d, unaligned, A, B, C);
constexpr uint8_t Z = 0x80;
alignas(16) static constexpr uint8_t kIdx_v0A[16] = {
0, 3, 6, 9, 12, 15, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0B[16] = {
Z, Z, Z, Z, Z, Z, 2, 5, 8, 11, 14, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 1, 4, 7, 10, 13};
alignas(16) static constexpr uint8_t kIdx_v1A[16] = {
1, 4, 7, 10, 13, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1B[16] = {
Z, Z, Z, Z, Z, 0, 3, 6, 9, 12, 15, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 2, 5, 8, 11, 14};
alignas(16) static constexpr uint8_t kIdx_v2A[16] = {
2, 5, 8, 11, 14, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2B[16] = {
Z, Z, Z, Z, Z, 1, 4, 7, 10, 13, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 0, 3, 6, 9, 12, 15};
const V v0L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v0A)));
const V v0M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v0B)));
const V v0U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v0C)));
const V v1L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v1A)));
const V v1M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v1B)));
const V v1U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v1C)));
const V v2L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v2A)));
const V v2M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v2B)));
const V v2U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v2C)));
v0 = Xor3(v0L, v0M, v0U);
v1 = Xor3(v1L, v1M, v1U);
v2 = Xor3(v2L, v2M, v2U);
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (8)>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
const RebindToUnsigned<decltype(d)> du;
using V = VFromD<D>;
V A;
V B;
V C;
detail::LoadTransposedBlocks3(d, unaligned, A, B, C);
constexpr uint8_t Z = 0x80;
alignas(16) static constexpr uint8_t kIdx_v0A[16] = {0, 3, 6, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0B[16] = {Z, Z, Z, 1, 4, 7, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0C[16] = {Z, Z, Z, Z, Z, Z, 2, 5};
alignas(16) static constexpr uint8_t kIdx_v1A[16] = {1, 4, 7, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1B[16] = {Z, Z, Z, 2, 5, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1C[16] = {Z, Z, Z, Z, Z, 0, 3, 6};
alignas(16) static constexpr uint8_t kIdx_v2A[16] = {2, 5, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2B[16] = {Z, Z, 0, 3, 6, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2C[16] = {Z, Z, Z, Z, Z, 1, 4, 7};
const V v0L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v0A)));
const V v0M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v0B)));
const V v0U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v0C)));
const V v1L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v1A)));
const V v1M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v1B)));
const V v1U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v1C)));
const V v2L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v2A)));
const V v2M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v2B)));
const V v2U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v2C)));
v0 = Xor3(v0L, v0M, v0U);
v1 = Xor3(v1L, v1M, v1U);
v2 = Xor3(v2L, v2M, v2U);
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (8)>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
const RebindToUnsigned<decltype(d)> du;
const Repartition<uint8_t, decltype(du)> du8;
using V = VFromD<D>;
V A;
V B;
V C;
detail::LoadTransposedBlocks3(d, unaligned, A, B, C);
constexpr uint8_t Z = 0x80;
alignas(16) static constexpr uint8_t kIdx_v0A[16] = {
0x00, 0x01, 0x06, 0x07, 0x0C, 0x0D, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0B[16] = {
Z, Z, Z, Z, Z, Z, 0x02, 0x03, 0x08, 0x09, 0x0E, 0x0F, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 0x04, 0x05, 0x0A, 0x0B};
alignas(16) static constexpr uint8_t kIdx_v1A[16] = {
0x02, 0x03, 0x08, 0x09, 0x0E, 0x0F, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1B[16] = {
Z, Z, Z, Z, Z, Z, 0x04, 0x05, 0x0A, 0x0B, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 0x00, 0x01, 0x06, 0x07, 0x0C, 0x0D};
alignas(16) static constexpr uint8_t kIdx_v2A[16] = {
0x04, 0x05, 0x0A, 0x0B, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2B[16] = {
Z, Z, Z, Z, 0x00, 0x01, 0x06, 0x07, 0x0C, 0x0D, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 0x02, 0x03, 0x08, 0x09, 0x0E, 0x0F};
const V v0L = TableLookupBytesOr0(A, BitCast(d, LoadDup128(du8, kIdx_v0A)));
const V v0M = TableLookupBytesOr0(B, BitCast(d, LoadDup128(du8, kIdx_v0B)));
const V v0U = TableLookupBytesOr0(C, BitCast(d, LoadDup128(du8, kIdx_v0C)));
const V v1L = TableLookupBytesOr0(A, BitCast(d, LoadDup128(du8, kIdx_v1A)));
const V v1M = TableLookupBytesOr0(B, BitCast(d, LoadDup128(du8, kIdx_v1B)));
const V v1U = TableLookupBytesOr0(C, BitCast(d, LoadDup128(du8, kIdx_v1C)));
const V v2L = TableLookupBytesOr0(A, BitCast(d, LoadDup128(du8, kIdx_v2A)));
const V v2M = TableLookupBytesOr0(B, BitCast(d, LoadDup128(du8, kIdx_v2B)));
const V v2U = TableLookupBytesOr0(C, BitCast(d, LoadDup128(du8, kIdx_v2C)));
v0 = Xor3(v0L, v0M, v0U);
v1 = Xor3(v1L, v1M, v1U);
v2 = Xor3(v2L, v2M, v2U);
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
using V = VFromD<D>;
V A;
V B;
V C;
detail::LoadTransposedBlocks3(d, unaligned, A, B, C);
const V vxx_02_03_xx = OddEven(C, B);
v0 = detail::ShuffleTwo1230(A, vxx_02_03_xx);
const V vxx_xx_10_11 = OddEven(A, B);
const V v12_13_xx_xx = OddEven(B, C);
v1 = detail::ShuffleTwo2301(vxx_xx_10_11, v12_13_xx_xx);
const V vxx_20_21_xx = OddEven(B, A);
v2 = detail::ShuffleTwo3012(vxx_20_21_xx, C);
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
VFromD<D> A;
VFromD<D> B;
VFromD<D> C;
detail::LoadTransposedBlocks3(d, unaligned, A, B, C);
v0 = OddEven(B, A);
v1 = CombineShiftRightBytes<sizeof(TFromD<D>)>(d, C, A);
v2 = OddEven(C, B);
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const T* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
v0 = LoadU(d, unaligned + 0);
v1 = LoadU(d, unaligned + 1);
v2 = LoadU(d, unaligned + 2);
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) void LoadTransposedBlocks4(D d,
const TFromD<D>* __restrict__ unaligned,
VFromD<D>& vA, VFromD<D>& vB,
VFromD<D>& vC, VFromD<D>& vD) {
constexpr size_t kN = MaxLanes(d);
vA = LoadU(d, unaligned + 0 * kN);
vB = LoadU(d, unaligned + 1 * kN);
vC = LoadU(d, unaligned + 2 * kN);
vD = LoadU(d, unaligned + 3 * kN);
}
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (16)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved4(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2,
VFromD<D>& v3) {
const Repartition<uint64_t, decltype(d)> d64;
using V64 = VFromD<decltype(d64)>;
using V = VFromD<D>;
V vA;
V vB;
V vC;
V vD;
detail::LoadTransposedBlocks4(d, unaligned, vA, vB, vC, vD);
const V v5140 = InterleaveLower(d, vA, vB);
const V vd9c8 = InterleaveLower(d, vC, vD);
const V v7362 = InterleaveUpper(d, vA, vB);
const V vfbea = InterleaveUpper(d, vC, vD);
const V v6420 = InterleaveLower(d, v5140, v7362);
const V veca8 = InterleaveLower(d, vd9c8, vfbea);
const V v7531 = InterleaveUpper(d, v5140, v7362);
const V vfdb9 = InterleaveUpper(d, vd9c8, vfbea);
const V64 v10L = BitCast(d64, InterleaveLower(d, v6420, v7531));
const V64 v10U = BitCast(d64, InterleaveLower(d, veca8, vfdb9));
const V64 v32L = BitCast(d64, InterleaveUpper(d, v6420, v7531));
const V64 v32U = BitCast(d64, InterleaveUpper(d, veca8, vfdb9));
v0 = BitCast(d, InterleaveLower(d64, v10L, v10U));
v1 = BitCast(d, InterleaveUpper(d64, v10L, v10U));
v2 = BitCast(d, InterleaveLower(d64, v32L, v32U));
v3 = BitCast(d, InterleaveUpper(d64, v32L, v32U));
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved4(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2,
VFromD<D>& v3) {
using TW = hwy::UnsignedFromSize<d.MaxBytes() == 8 ? 4 : 8>;
const Repartition<TW, decltype(d)> dw;
using VW = VFromD<decltype(dw)>;
VFromD<D> vA;
VFromD<D> vB;
VFromD<D> vC;
VFromD<D> vD;
detail::LoadTransposedBlocks4(d, unaligned, vA, vB, vC, vD);
const VFromD<D> va820 = InterleaveLower(d, vA, vB);
const VFromD<D> vec64 = InterleaveLower(d, vC, vD);
const VFromD<D> vb931 = InterleaveUpper(d, vA, vB);
const VFromD<D> vfd75 = InterleaveUpper(d, vC, vD);
const VW v10_b830 =
BitCast(dw, InterleaveLower(d, va820, vb931));
const VW v10_fc74 =
BitCast(dw, InterleaveLower(d, vec64, vfd75));
const VW v32_b830 =
BitCast(dw, InterleaveUpper(d, va820, vb931));
const VW v32_fc74 =
BitCast(dw, InterleaveUpper(d, vec64, vfd75));
v0 = BitCast(d, InterleaveLower(dw, v10_b830, v10_fc74));
v1 = BitCast(d, InterleaveUpper(dw, v10_b830, v10_fc74));
v2 = BitCast(d, InterleaveLower(dw, v32_b830, v32_fc74));
v3 = BitCast(d, InterleaveUpper(dw, v32_b830, v32_fc74));
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved4(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2,
VFromD<D>& v3) {
using V = VFromD<D>;
V vA;
V vB;
V vC;
V vD;
detail::LoadTransposedBlocks4(d, unaligned, vA, vB, vC, vD);
const V v10e = InterleaveLower(d, vA, vC);
const V v10o = InterleaveLower(d, vB, vD);
const V v32e = InterleaveUpper(d, vA, vC);
const V v32o = InterleaveUpper(d, vB, vD);
v0 = InterleaveLower(d, v10e, v10o);
v1 = InterleaveUpper(d, v10e, v10o);
v2 = InterleaveLower(d, v32e, v32o);
v3 = InterleaveUpper(d, v32e, v32o);
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved4(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2,
VFromD<D>& v3) {
VFromD<D> vA, vB, vC, vD;
detail::LoadTransposedBlocks4(d, unaligned, vA, vB, vC, vD);
v0 = InterleaveLower(d, vA, vC);
v1 = InterleaveUpper(d, vA, vC);
v2 = InterleaveLower(d, vB, vD);
v3 = InterleaveUpper(d, vB, vD);
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved4(D d, const T* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2,
VFromD<D>& v3) {
v0 = LoadU(d, unaligned + 0);
v1 = LoadU(d, unaligned + 1);
v2 = LoadU(d, unaligned + 2);
v3 = LoadU(d, unaligned + 3);
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) void StoreTransposedBlocks2(VFromD<D> A, VFromD<D> B, D d,
TFromD<D>* __restrict__ unaligned) {
constexpr size_t kN = MaxLanes(d);
StoreU(A, d, unaligned + 0 * kN);
StoreU(B, d, unaligned + 1 * kN);
}
}
template <class D, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved2(VFromD<D> v0, VFromD<D> v1, D d,
TFromD<D>* __restrict__ unaligned) {
const auto v10L = InterleaveLower(d, v0, v1);
const auto v10U = InterleaveUpper(d, v0, v1);
detail::StoreTransposedBlocks2(v10L, v10U, d, unaligned);
}
template <class V, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved2(V part0, V part1, D d,
TFromD<D>* __restrict__ unaligned) {
const Twice<decltype(d)> d2;
const auto v0 = ZeroExtendVector(d2, part0);
const auto v1 = ZeroExtendVector(d2, part1);
const auto v10 = InterleaveLower(d2, v0, v1);
StoreU(v10, d2, unaligned);
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) void StoreTransposedBlocks3(VFromD<D> A, VFromD<D> B, VFromD<D> C,
D d, TFromD<D>* __restrict__ unaligned) {
constexpr size_t kN = MaxLanes(d);
StoreU(A, d, unaligned + 0 * kN);
StoreU(B, d, unaligned + 1 * kN);
StoreU(C, d, unaligned + 2 * kN);
}
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
const auto k5 = Set(du, TU{5});
const auto k6 = Set(du, TU{6});
alignas(16) static constexpr uint8_t tbl_v0[16] = {
0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80, 0x80,
3, 0x80, 0x80, 4, 0x80, 0x80, 5};
alignas(16) static constexpr uint8_t tbl_v1[16] = {
0x80, 0, 0x80, 0x80, 1, 0x80,
0x80, 2, 0x80, 0x80, 3, 0x80, 0x80, 4, 0x80, 0x80};
const auto shuf_A0 = LoadDup128(du, tbl_v0);
const auto shuf_A1 = LoadDup128(du, tbl_v1);
const auto shuf_A2 = CombineShiftRightBytes<15>(du, shuf_A1, shuf_A1);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const VFromD<D> A = BitCast(d, A0 | A1 | A2);
const auto shuf_B0 = shuf_A2 + k6;
const auto shuf_B1 = shuf_A0 + k5;
const auto shuf_B2 = shuf_A1 + k5;
const auto B0 = TableLookupBytesOr0(v0, shuf_B0);
const auto B1 = TableLookupBytesOr0(v1, shuf_B1);
const auto B2 = TableLookupBytesOr0(v2, shuf_B2);
const VFromD<D> B = BitCast(d, B0 | B1 | B2);
const auto shuf_C0 = shuf_B2 + k6;
const auto shuf_C1 = shuf_B0 + k5;
const auto shuf_C2 = shuf_B1 + k5;
const auto C0 = TableLookupBytesOr0(v0, shuf_C0);
const auto C1 = TableLookupBytesOr0(v1, shuf_C1);
const auto C2 = TableLookupBytesOr0(v2, shuf_C2);
const VFromD<D> C = BitCast(d, C0 | C1 | C2);
detail::StoreTransposedBlocks3(A, B, C, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
const Repartition<uint8_t, decltype(d)> du8;
const auto k2 = Set(du8, uint8_t{2 * sizeof(TFromD<D>)});
const auto k3 = Set(du8, uint8_t{3 * sizeof(TFromD<D>)});
alignas(16) static constexpr uint8_t tbl_v1[16] = {
0x80, 0x80, 0, 1, 0x80, 0x80, 0x80, 0x80,
2, 3, 0x80, 0x80, 0x80, 0x80, 4, 5};
alignas(16) static constexpr uint8_t tbl_v2[16] = {
0x80, 0x80, 0x80, 0x80, 0, 1, 0x80, 0x80,
0x80, 0x80, 2, 3, 0x80, 0x80, 0x80, 0x80};
const auto shuf_A1 = LoadDup128(du8, tbl_v1);
const auto shuf_A0 = CombineShiftRightBytes<2>(du8, shuf_A1, shuf_A1);
const auto shuf_A2 = LoadDup128(du8, tbl_v2);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const VFromD<D> A = BitCast(d, A0 | A1 | A2);
const auto shuf_B0 = shuf_A1 + k3;
const auto shuf_B1 = shuf_A2 + k3;
const auto shuf_B2 = shuf_A0 + k2;
const auto B0 = TableLookupBytesOr0(v0, shuf_B0);
const auto B1 = TableLookupBytesOr0(v1, shuf_B1);
const auto B2 = TableLookupBytesOr0(v2, shuf_B2);
const VFromD<D> B = BitCast(d, B0 | B1 | B2);
const auto shuf_C0 = shuf_B1 + k3;
const auto shuf_C1 = shuf_B2 + k3;
const auto shuf_C2 = shuf_B0 + k2;
const auto C0 = TableLookupBytesOr0(v0, shuf_C0);
const auto C1 = TableLookupBytesOr0(v1, shuf_C1);
const auto C2 = TableLookupBytesOr0(v2, shuf_C2);
const VFromD<D> C = BitCast(d, C0 | C1 | C2);
detail::StoreTransposedBlocks3(A, B, C, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
const RepartitionToWide<decltype(d)> dw;
const VFromD<D> v10_v00 = InterleaveLower(d, v0, v1);
const VFromD<D> v01_v20 = OddEven(v0, v2);
const VFromD<D> A = BitCast(
d, InterleaveLower(dw, BitCast(dw, v10_v00), BitCast(dw, v01_v20)));
const VFromD<D> v1_321 = ShiftRightLanes<1>(d, v1);
const VFromD<D> v0_32 = ShiftRightLanes<2>(d, v0);
const VFromD<D> v21_v11 = OddEven(v2, v1_321);
const VFromD<D> v12_v02 = OddEven(v1_321, v0_32);
const VFromD<D> B = BitCast(
d, InterleaveLower(dw, BitCast(dw, v21_v11), BitCast(dw, v12_v02)));
const VFromD<D> v23_v13 = OddEven(v2, v1_321);
const VFromD<D> v03_v22 = OddEven(v0, v2);
const VFromD<D> C = BitCast(
d, InterleaveUpper(dw, BitCast(dw, v03_v22), BitCast(dw, v23_v13)));
detail::StoreTransposedBlocks3(A, B, C, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
const VFromD<D> A = InterleaveLower(d, v0, v1);
const VFromD<D> B = OddEven(v0, v2);
const VFromD<D> C = InterleaveUpper(d, v1, v2);
detail::StoreTransposedBlocks3(A, B, C, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, D d,
TFromD<D>* __restrict__ unaligned) {
constexpr size_t kFullN = 16 / sizeof(TFromD<D>);
const Full128<uint8_t> du;
const Full128<TFromD<D>> d_full;
const auto k5 = Set(du, uint8_t{5});
const auto k6 = Set(du, uint8_t{6});
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
alignas(16) static constexpr uint8_t tbl_v0[16] = {
0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80, 0x80,
3, 0x80, 0x80, 4, 0x80, 0x80, 5};
alignas(16) static constexpr uint8_t tbl_v1[16] = {
0x80, 0, 0x80, 0x80, 1, 0x80,
0x80, 2, 0x80, 0x80, 3, 0x80, 0x80, 4, 0x80, 0x80};
const auto shuf_A0 = Load(du, tbl_v0);
const auto shuf_A1 = Load(du, tbl_v1);
const auto shuf_A2 = CombineShiftRightBytes<15>(du, shuf_A1, shuf_A1);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const auto A = BitCast(d_full, A0 | A1 | A2);
StoreU(A, d_full, unaligned + 0 * kFullN);
const auto shuf_B0 = shuf_A2 + k6;
const auto shuf_B1 = shuf_A0 + k5;
const auto shuf_B2 = shuf_A1 + k5;
const auto B0 = TableLookupBytesOr0(v0, shuf_B0);
const auto B1 = TableLookupBytesOr0(v1, shuf_B1);
const auto B2 = TableLookupBytesOr0(v2, shuf_B2);
const VFromD<D> B{BitCast(d_full, B0 | B1 | B2).raw};
StoreU(B, d, unaligned + 1 * kFullN);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, D dh,
TFromD<D>* __restrict__ unaligned) {
const Twice<D> d_full;
const Full128<uint8_t> du8;
const auto k2 = Set(du8, uint8_t{2 * sizeof(TFromD<D>)});
const auto k3 = Set(du8, uint8_t{3 * sizeof(TFromD<D>)});
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
alignas(16) static constexpr uint8_t tbl_v1[16] = {
0x80, 0x80, 0, 1, 0x80, 0x80, 0x80, 0x80,
2, 3, 0x80, 0x80, 0x80, 0x80, 4, 5};
alignas(16) static constexpr uint8_t tbl_v2[16] = {
0x80, 0x80, 0x80, 0x80, 0, 1, 0x80, 0x80,
0x80, 0x80, 2, 3, 0x80, 0x80, 0x80, 0x80};
const auto shuf_A1 = Load(du8, tbl_v1);
const auto shuf_A0 = CombineShiftRightBytes<2>(du8, shuf_A1, shuf_A1);
const auto shuf_A2 = Load(du8, tbl_v2);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const VFromD<decltype(d_full)> A = BitCast(d_full, A0 | A1 | A2);
StoreU(A, d_full, unaligned);
const auto shuf_B0 = shuf_A1 + k3;
const auto shuf_B1 = shuf_A2 + k3;
const auto shuf_B2 = shuf_A0 + k2;
const auto B0 = TableLookupBytesOr0(v0, shuf_B0);
const auto B1 = TableLookupBytesOr0(v1, shuf_B1);
const auto B2 = TableLookupBytesOr0(v2, shuf_B2);
const VFromD<decltype(d_full)> B = BitCast(d_full, B0 | B1 | B2);
StoreU(VFromD<D>{B.raw}, dh, unaligned + MaxLanes(d_full));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
const VFromD<D> v10_v00 = InterleaveLower(d, v0, v1);
const VFromD<D> v01_v20 = OddEven(v0, v2);
const VFromD<D> v21_v11 = InterleaveUpper(d, v1, v2);
constexpr size_t kN = MaxLanes(d);
StoreU(v10_v00, d, unaligned + 0 * kN);
StoreU(v01_v20, d, unaligned + 1 * kN);
StoreU(v21_v11, d, unaligned + 2 * kN);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr,
hwy::EnableIf<(D::kPrivateLanes > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, D d,
TFromD<D>* __restrict__ unaligned) {
const Full128<uint8_t> du;
const Full128<TFromD<D>> d_full;
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
alignas(16) static constexpr uint8_t tbl_v0[16] = {
0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80,
0x80, 3, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};
const auto shuf_A0 = Load(du, tbl_v0);
const auto shuf_A1 = CombineShiftRightBytes<15>(du, shuf_A0, shuf_A0);
const auto shuf_A2 = CombineShiftRightBytes<14>(du, shuf_A0, shuf_A0);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const VFromD<decltype(d_full)> A = BitCast(d_full, A0 | A1 | A2);
alignas(16) TFromD<D> buf[MaxLanes(d_full)];
StoreU(A, d_full, buf);
CopyBytes<d.MaxBytes() * 3>(buf, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, D d,
TFromD<D>* __restrict__ unaligned) {
const Full128<uint8_t> du8;
const Full128<TFromD<D>> d_full;
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
alignas(16) static constexpr uint8_t tbl_v2[16] = {
0x80, 0x80, 0x80, 0x80, 0, 1, 0x80, 0x80,
0x80, 0x80, 2, 3, 0x80, 0x80, 0x80, 0x80};
const auto shuf_A2 =
Load(du8, tbl_v2);
const auto shuf_A1 =
CombineShiftRightBytes<2>(du8, shuf_A2, shuf_A2);
const auto shuf_A0 =
CombineShiftRightBytes<4>(du8, shuf_A2, shuf_A2);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const auto A = BitCast(d_full, A0 | A1 | A2);
alignas(16) TFromD<D> buf[MaxLanes(d_full)];
StoreU(A, d_full, buf);
CopyBytes<d.MaxBytes() * 3>(buf, unaligned);
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
StoreU(v0, d, unaligned + 0);
StoreU(v1, d, unaligned + 1);
StoreU(v2, d, unaligned + 2);
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) void StoreTransposedBlocks4(VFromD<D> vA, VFromD<D> vB, VFromD<D> vC,
VFromD<D> vD, D d,
TFromD<D>* __restrict__ unaligned) {
constexpr size_t kN = MaxLanes(d);
StoreU(vA, d, unaligned + 0 * kN);
StoreU(vB, d, unaligned + 1 * kN);
StoreU(vC, d, unaligned + 2 * kN);
StoreU(vD, d, unaligned + 3 * kN);
}
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) != (8)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved4(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2,
VFromD<D> v3, D d,
TFromD<D>* __restrict__ unaligned) {
const RepartitionToWide<decltype(d)> dw;
const auto v10L = ZipLower(dw, v0, v1);
const auto v32L = ZipLower(dw, v2, v3);
const auto v10U = ZipUpper(dw, v0, v1);
const auto v32U = ZipUpper(dw, v2, v3);
const VFromD<D> vA = BitCast(d, InterleaveLower(dw, v10L, v32L));
const VFromD<D> vB = BitCast(d, InterleaveUpper(dw, v10L, v32L));
const VFromD<D> vC = BitCast(d, InterleaveLower(dw, v10U, v32U));
const VFromD<D> vD = BitCast(d, InterleaveUpper(dw, v10U, v32U));
detail::StoreTransposedBlocks4(vA, vB, vC, vD, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved4(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2,
VFromD<D> v3, D d,
TFromD<D>* __restrict__ unaligned) {
const VFromD<D> vA = InterleaveLower(d, v0, v1);
const VFromD<D> vB = InterleaveLower(d, v2, v3);
const VFromD<D> vC = InterleaveUpper(d, v0, v1);
const VFromD<D> vD = InterleaveUpper(d, v2, v3);
detail::StoreTransposedBlocks4(vA, vB, vC, vD, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) != (8)>* = nullptr, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved4(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, VFromD<D> part3, D ,
TFromD<D>* __restrict__ unaligned) {
const Full128<TFromD<D>> d_full;
const RepartitionToWide<decltype(d_full)> dw;
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
const VFromD<decltype(d_full)> v3{part3.raw};
const auto v10 = ZipLower(dw, v0, v1);
const auto v32 = ZipLower(dw, v2, v3);
const auto A = BitCast(d_full, InterleaveLower(dw, v10, v32));
const auto B = BitCast(d_full, InterleaveUpper(dw, v10, v32));
StoreU(A, d_full, unaligned);
StoreU(B, d_full, unaligned + MaxLanes(d_full));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved4(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, VFromD<D> part3, D ,
TFromD<D>* __restrict__ unaligned) {
const Full128<TFromD<D>> d_full;
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
const VFromD<decltype(d_full)> v3{part3.raw};
const auto A = InterleaveLower(d_full, v0, v1);
const auto B = InterleaveLower(d_full, v2, v3);
StoreU(A, d_full, unaligned);
StoreU(B, d_full, unaligned + MaxLanes(d_full));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved4(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, VFromD<D> part3, D d,
TFromD<D>* __restrict__ unaligned) {
const Full128<TFromD<D>> d_full;
const RepartitionToWide<decltype(d_full)> dw;
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
const VFromD<decltype(d_full)> v3{part3.raw};
const auto v10 = ZipLower(dw, v0, v1);
const auto v32 = ZipLower(dw, v2, v3);
const auto v3210 = BitCast(d_full, InterleaveLower(dw, v10, v32));
alignas(16) TFromD<D> buf[MaxLanes(d_full)];
StoreU(v3210, d_full, buf);
CopyBytes<d.MaxBytes() * 4>(buf, unaligned);
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AbsDiff(V a, V b) {
return Sub(Max(a, b), Min(a, b));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedAdd(V a, V b) {
const DFromV<decltype(a)> d;
const auto sum = Add(a, b);
const auto overflow_mask =
MaskFromVec(BroadcastSignBit(AndNot(Xor(a, b), Xor(a, sum))));
const auto overflow_result =
Xor(BroadcastSignBit(a), Set(d, LimitsMax<int32_t>()));
return IfThenElse(overflow_mask, overflow_result, sum);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedSub(V a, V b) {
const DFromV<decltype(a)> d;
const auto diff = Sub(a, b);
const auto overflow_mask =
MaskFromVec(BroadcastSignBit(And(Xor(a, b), Xor(a, diff))));
const auto overflow_result =
Xor(BroadcastSignBit(a), Set(d, LimitsMax<int32_t>()));
return IfThenElse(overflow_mask, overflow_result, diff);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedAdd(V a, V b) {
const DFromV<decltype(a)> d;
const auto sum = Add(a, b);
const auto overflow_mask =
MaskFromVec(BroadcastSignBit(AndNot(Xor(a, b), Xor(a, sum))));
const auto overflow_result =
Xor(BroadcastSignBit(a), Set(d, LimitsMax<int64_t>()));
return IfThenElse(overflow_mask, overflow_result, sum);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedSub(V a, V b) {
const DFromV<decltype(a)> d;
const auto diff = Sub(a, b);
const auto overflow_mask =
MaskFromVec(BroadcastSignBit(And(Xor(a, b), Xor(a, diff))));
const auto overflow_result =
Xor(BroadcastSignBit(a), Set(d, LimitsMax<int64_t>()));
return IfThenElse(overflow_mask, overflow_result, diff);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedAdd(V a, V b) {
return Add(a, Min(b, Not(a)));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedSub(V a, V b) {
return Sub(a, Min(a, b));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedAdd(V a, V b) {
return Add(a, Min(b, Not(a)));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedSub(V a, V b) {
return Sub(a, Min(a, b));
}
template <class DN, hwy::EnableIf<IsSigned<TFromD<DN> >() && !IsFloat<TFromD<DN> >() && !IsSpecialFloat<TFromD<DN> >()>* = nullptr, class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
class V2 = VFromD<Rebind<TFromV<V>, DN>>,
hwy::EnableIf<(sizeof(TFromD<DN>) < sizeof(TFromV<V>))>* = nullptr,
hwy::EnableIf<(DFromV<V>::kPrivateLanes == DFromV<V2>::kPrivateLanes)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DN> DemoteTo(DN dn, V v) {
const DFromV<decltype(v)> d;
const RebindToSigned<decltype(d)> di;
const RebindToUnsigned<decltype(dn)> dn_u;
const auto i2i_demote_result = DemoteTo(dn, BitCast(di, v));
const auto max_signed_val = Set(dn, hwy::HighestValue<TFromD<DN>>());
return BitCast(
dn, Min(BitCast(dn_u, i2i_demote_result), BitCast(dn_u, max_signed_val)));
}
template <class DN, hwy::EnableIf<IsSigned<TFromD<DN> >() && !IsFloat<TFromD<DN> >() && !IsSpecialFloat<TFromD<DN> >()>* = nullptr, class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
class V2 = VFromD<Repartition<TFromV<V>, DN>>,
hwy::EnableIf<sizeof(TFromV<V>) == (sizeof(TFromD<DN>) * 2)>* = nullptr,
hwy::EnableIf<(DFromV<V>::kPrivateLanes == DFromV<V2>::kPrivateLanes)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DN> ReorderDemote2To(DN dn, V a, V b) {
const DFromV<decltype(a)> d;
const RebindToSigned<decltype(d)> di;
const RebindToUnsigned<decltype(dn)> dn_u;
const auto i2i_demote_result =
ReorderDemote2To(dn, BitCast(di, a), BitCast(di, b));
const auto max_signed_val = Set(dn, hwy::HighestValue<TFromD<DN>>());
return BitCast(
dn, Min(BitCast(dn_u, i2i_demote_result), BitCast(dn_u, max_signed_val)));
}
template <class DN, hwy::EnableIf<!IsSigned<TFromD<DN> >()>* = nullptr, class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
hwy::EnableIf<sizeof(TFromV<V>) == (sizeof(TFromD<DN>) * 2)>* = nullptr,
hwy::EnableIf<(DFromV<VFromD<DN>>::kPrivateLanes == DFromV<V>::kPrivateLanes * 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DN> OrderedTruncate2To(DN dn, V a, V b) {
return ConcatEven(dn, BitCast(dn, b), BitCast(dn, a));
}
namespace detail {
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const RebindToFloat<decltype(d)> df;
const RebindToSigned<decltype(d)> di;
const Repartition<int16_t, decltype(d)> di16;
const auto f32_bits = BitCast(d, ConvertTo(df, BitCast(di, v)));
return BitCast(d, Min(BitCast(di16, ShiftRight<23>(f32_bits)),
BitCast(di16, Set(d, 158))));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint32_t>()>* = nullptr>
inline __attribute__((always_inline)) V I32RangeU32ToF32BiasedExp(V v) {
const DFromV<decltype(v)> d;
const RebindToFloat<decltype(d)> df;
const RebindToSigned<decltype(d)> d_src;
const auto f32_bits = BitCast(d, ConvertTo(df, BitCast(d_src, v)));
return ShiftRight<23>(f32_bits);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes <= 16 / 4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Rebind<uint32_t, decltype(d)> du32;
const auto f32_biased_exp_as_u32 =
I32RangeU32ToF32BiasedExp(PromoteTo(du32, v));
return TruncateTo(d, f32_biased_exp_as_u32);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 16 / 4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Half<decltype(d)> dh;
const Rebind<uint32_t, decltype(dh)> du32;
const auto lo_u32 = PromoteTo(du32, LowerHalf(dh, v));
const auto hi_u32 = PromoteTo(du32, UpperHalf(dh, v));
const auto lo_f32_biased_exp_as_u32 = I32RangeU32ToF32BiasedExp(lo_u32);
const auto hi_f32_biased_exp_as_u32 = I32RangeU32ToF32BiasedExp(hi_u32);
const RebindToSigned<decltype(du32)> di32;
const RebindToSigned<decltype(d)> di;
return BitCast(d,
OrderedDemote2To(di, BitCast(di32, lo_f32_biased_exp_as_u32),
BitCast(di32, hi_f32_biased_exp_as_u32)));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes <= 16 / 4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Rebind<uint32_t, decltype(d)> du32;
const auto f32_biased_exp_as_u32 =
I32RangeU32ToF32BiasedExp(PromoteTo(du32, v));
return U8FromU32(f32_biased_exp_as_u32);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 16 / 4)>* = nullptr,
hwy::EnableIf<(D::kPrivateLanes <= 16 / 2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Half<decltype(d)> dh;
const Rebind<uint32_t, decltype(dh)> du32;
const Repartition<uint16_t, decltype(du32)> du16;
const auto lo_u32 = PromoteTo(du32, LowerHalf(dh, v));
const auto hi_u32 = PromoteTo(du32, UpperHalf(dh, v));
const auto lo_f32_biased_exp_as_u32 = I32RangeU32ToF32BiasedExp(lo_u32);
const auto hi_f32_biased_exp_as_u32 = I32RangeU32ToF32BiasedExp(hi_u32);
const RebindToSigned<decltype(du32)> di32;
const RebindToSigned<decltype(du16)> di16;
const auto f32_biased_exp_as_i16 =
OrderedDemote2To(di16, BitCast(di32, lo_f32_biased_exp_as_u32),
BitCast(di32, hi_f32_biased_exp_as_u32));
return DemoteTo(d, f32_biased_exp_as_i16);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 16 / 2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Half<decltype(d)> dh;
const Half<decltype(dh)> dq;
const Rebind<uint32_t, decltype(dq)> du32;
const Repartition<uint16_t, decltype(du32)> du16;
const auto lo_half = LowerHalf(dh, v);
const auto hi_half = UpperHalf(dh, v);
const auto u32_q0 = PromoteTo(du32, LowerHalf(dq, lo_half));
const auto u32_q1 = PromoteTo(du32, UpperHalf(dq, lo_half));
const auto u32_q2 = PromoteTo(du32, LowerHalf(dq, hi_half));
const auto u32_q3 = PromoteTo(du32, UpperHalf(dq, hi_half));
const auto f32_biased_exp_as_u32_q0 = I32RangeU32ToF32BiasedExp(u32_q0);
const auto f32_biased_exp_as_u32_q1 = I32RangeU32ToF32BiasedExp(u32_q1);
const auto f32_biased_exp_as_u32_q2 = I32RangeU32ToF32BiasedExp(u32_q2);
const auto f32_biased_exp_as_u32_q3 = I32RangeU32ToF32BiasedExp(u32_q3);
const RebindToSigned<decltype(du32)> di32;
const RebindToSigned<decltype(du16)> di16;
const auto lo_f32_biased_exp_as_i16 =
OrderedDemote2To(di16, BitCast(di32, f32_biased_exp_as_u32_q0),
BitCast(di32, f32_biased_exp_as_u32_q1));
const auto hi_f32_biased_exp_as_i16 =
OrderedDemote2To(di16, BitCast(di32, f32_biased_exp_as_u32_q2),
BitCast(di32, f32_biased_exp_as_u32_q3));
return OrderedDemote2To(d, lo_f32_biased_exp_as_i16,
hi_f32_biased_exp_as_i16);
}
template <class D>
using F32ExpLzcntMinMaxRepartition = Repartition<uint8_t, D>;
template <class V>
using F32ExpLzcntMinMaxCmpV = VFromD<F32ExpLzcntMinMaxRepartition<DFromV<V>>>;
template <class V>
inline __attribute__((always_inline)) F32ExpLzcntMinMaxCmpV<V> F32ExpLzcntMinMaxBitCast(V v) {
const DFromV<decltype(v)> d;
const F32ExpLzcntMinMaxRepartition<decltype(d)> d2;
return BitCast(d2, v);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Repartition<uint32_t, decltype(d)> du32;
const auto f32_biased_exp = UIntToF32BiasedExp(du32, BitCast(du32, v));
const auto f32_biased_exp_adj =
IfThenZeroElse(Eq(f32_biased_exp, Zero(du32)),
BitCast(du32, Set(d, 0x0000002000000000u)));
const auto adj_f32_biased_exp = Add(f32_biased_exp, f32_biased_exp_adj);
return ShiftRight<32>(BitCast(
d, Max(F32ExpLzcntMinMaxBitCast(adj_f32_biased_exp),
F32ExpLzcntMinMaxBitCast(Reverse2(du32, adj_f32_biased_exp)))));
}
template <class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr>
inline __attribute__((always_inline)) V UIntToF32BiasedExp(V v) {
const DFromV<decltype(v)> d;
return UIntToF32BiasedExp(d, v);
}
template <class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2))) != 0>* = nullptr>
inline __attribute__((always_inline)) V NormalizeForUIntTruncConvToF32(V v) {
return v;
}
template <class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 4) | (1 << 8))) != 0>* = nullptr>
inline __attribute__((always_inline)) V NormalizeForUIntTruncConvToF32(V v) {
return AndNot(ShiftRight<24>(v), v);
}
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V HighestSetBitIndex(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
const auto f32_biased_exp = detail::UIntToF32BiasedExp(
detail::NormalizeForUIntTruncConvToF32(BitCast(du, v)));
return BitCast(d, Sub(f32_biased_exp, Set(du, TU{127})));
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V LeadingZeroCount(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
constexpr TU kNumOfBitsInT{sizeof(TU) * 8};
const auto f32_biased_exp = detail::UIntToF32BiasedExp(
detail::NormalizeForUIntTruncConvToF32(BitCast(du, v)));
const auto lz_count = Sub(Set(du, TU{kNumOfBitsInT + 126}), f32_biased_exp);
return BitCast(d,
Min(detail::F32ExpLzcntMinMaxBitCast(lz_count),
detail::F32ExpLzcntMinMaxBitCast(Set(du, kNumOfBitsInT))));
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V TrailingZeroCount(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const RebindToSigned<decltype(d)> di;
using TU = TFromD<decltype(du)>;
const auto vi = BitCast(di, v);
const auto lowest_bit = BitCast(du, And(vi, Neg(vi)));
constexpr TU kNumOfBitsInT{sizeof(TU) * 8};
const auto f32_biased_exp = detail::UIntToF32BiasedExp(lowest_bit);
const auto tz_count = Sub(f32_biased_exp, Set(du, TU{127}));
return BitCast(d,
Min(detail::F32ExpLzcntMinMaxBitCast(tz_count),
detail::F32ExpLzcntMinMaxBitCast(Set(du, kNumOfBitsInT))));
}
namespace detail {
template <class V>
inline __attribute__((always_inline)) V SubBytesMulInverseAndAffineLookup(V state, V affine_tblL,
V affine_tblU) {
const DFromV<V> du;
const auto mask = Set(du, uint8_t{0xF});
{
alignas(16) static constexpr uint8_t basisL[16] = {
0x00, 0x70, 0x2A, 0x5A, 0x98, 0xE8, 0xB2, 0xC2,
0x08, 0x78, 0x22, 0x52, 0x90, 0xE0, 0xBA, 0xCA};
alignas(16) static constexpr uint8_t basisU[16] = {
0x00, 0x4D, 0x7C, 0x31, 0x7D, 0x30, 0x01, 0x4C,
0x81, 0xCC, 0xFD, 0xB0, 0xFC, 0xB1, 0x80, 0xCD};
const auto sL = And(state, mask);
const auto sU = ShiftRight<4>(state);
const auto gf4L = TableLookupBytes(LoadDup128(du, basisL), sL);
const auto gf4U = TableLookupBytes(LoadDup128(du, basisU), sU);
state = Xor(gf4L, gf4U);
}
alignas(16) static constexpr uint8_t kZetaInv[16] = {
0x80, 7, 11, 15, 6, 10, 4, 1, 9, 8, 5, 2, 12, 14, 13, 3};
alignas(16) static constexpr uint8_t kInv[16] = {
0x80, 1, 8, 13, 15, 6, 5, 14, 2, 12, 11, 10, 9, 3, 7, 4};
const auto tbl = LoadDup128(du, kInv);
const auto sL = And(state, mask);
const auto sU = ShiftRight<4>(state);
const auto sX = Xor(sU, sL);
const auto invL = TableLookupBytes(LoadDup128(du, kZetaInv), sL);
const auto invU = TableLookupBytes(tbl, sU);
const auto invX = TableLookupBytes(tbl, sX);
const auto outL = Xor(sX, TableLookupBytesOr0(tbl, Xor(invL, invU)));
const auto outU = Xor(sU, TableLookupBytesOr0(tbl, Xor(invL, invX)));
const auto affL = TableLookupBytesOr0(affine_tblL, outL);
const auto affU = TableLookupBytesOr0(affine_tblU, outU);
return Xor(affL, affU);
}
template <class V>
inline __attribute__((always_inline)) V SubBytes(V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t kAffineL[16] = {
0x00, 0xC7, 0xBD, 0x6F, 0x17, 0x6D, 0xD2, 0xD0,
0x78, 0xA8, 0x02, 0xC5, 0x7A, 0xBF, 0xAA, 0x15};
alignas(16) static constexpr uint8_t kAffineU[16] = {
0x00, 0x6A, 0xBB, 0x5F, 0xA5, 0x74, 0xE4, 0xCF,
0xFA, 0x35, 0x2B, 0x41, 0xD1, 0x90, 0x1E, 0x8E};
return Xor(SubBytesMulInverseAndAffineLookup(state, LoadDup128(du, kAffineL),
LoadDup128(du, kAffineU)),
Set(du, uint8_t{0x63}));
}
template <class V>
inline __attribute__((always_inline)) V InvSubBytes(V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t kGF2P4InvToGF2P8InvL[16]{
0x00, 0x40, 0xF9, 0x7E, 0x53, 0xEA, 0x87, 0x13,
0x2D, 0x3E, 0x94, 0xD4, 0xB9, 0x6D, 0xAA, 0xC7};
alignas(16) static constexpr uint8_t kGF2P4InvToGF2P8InvU[16]{
0x00, 0x1D, 0x44, 0x93, 0x0F, 0x56, 0xD7, 0x12,
0x9C, 0x8E, 0xC5, 0xD8, 0x59, 0x81, 0x4B, 0xCA};
const auto b = Xor(Xor3(Or(ShiftLeft<1>(state), ShiftRight<7>(state)),
Or(ShiftLeft<3>(state), ShiftRight<5>(state)),
Or(ShiftLeft<6>(state), ShiftRight<2>(state))),
Set(du, uint8_t{0x05}));
return SubBytesMulInverseAndAffineLookup(
b, LoadDup128(du, kGF2P4InvToGF2P8InvL),
LoadDup128(du, kGF2P4InvToGF2P8InvU));
}
}
namespace detail {
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ShiftRows(const V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t kShiftRow[16] = {
0, 5, 10, 15,
4, 9, 14, 3,
8, 13, 2, 7,
12, 1, 6, 11};
const auto shift_row = LoadDup128(du, kShiftRow);
return TableLookupBytes(state, shift_row);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V InvShiftRows(const V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t kShiftRow[16] = {
0, 13, 10, 7,
4, 1, 14, 11,
8, 5, 2, 15,
12, 9, 6, 3};
const auto shift_row = LoadDup128(du, kShiftRow);
return TableLookupBytes(state, shift_row);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V GF2P8Mod11BMulBy2(V v) {
const DFromV<V> du;
const RebindToSigned<decltype(du)> di;
const auto msb = Lt(BitCast(di, v), Zero(di));
const auto overflow = BitCast(du, IfThenElseZero(msb, Set(di, int8_t{0x1B})));
return Xor(Add(v, v), overflow);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V MixColumns(const V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t k2301[16] = {
2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13};
alignas(16) static constexpr uint8_t k1230[16] = {
1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12};
const auto d = GF2P8Mod11BMulBy2(state);
const auto s2301 = TableLookupBytes(state, LoadDup128(du, k2301));
const auto d_s2301 = Xor(d, s2301);
const auto t_s2301 = Xor(state, d_s2301);
const auto t1230_s3012 = TableLookupBytes(t_s2301, LoadDup128(du, k1230));
return Xor(d_s2301, t1230_s3012);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V InvMixColumns(const V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t k2301[16] = {
2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13};
alignas(16) static constexpr uint8_t k1230[16] = {
1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12};
const auto v1230 = LoadDup128(du, k1230);
const auto sx2 = GF2P8Mod11BMulBy2(state);
const auto sx4 = GF2P8Mod11BMulBy2(sx2);
const auto sx8 = GF2P8Mod11BMulBy2(sx4);
const auto sx9 = Xor(sx8, state);
const auto sx11 = Xor(sx9, sx2);
const auto sx13 = Xor(sx9, sx4);
const auto sx14 = Xor3(sx8, sx4, sx2);
const auto sx13_0123_sx9_1230 = Xor(sx13, TableLookupBytes(sx9, v1230));
const auto sx14_0123_sx11_1230 = Xor(sx14, TableLookupBytes(sx11, v1230));
const auto sx13_2301_sx9_3012 =
TableLookupBytes(sx13_0123_sx9_1230, LoadDup128(du, k2301));
return Xor(sx14_0123_sx11_1230, sx13_2301_sx9_3012);
}
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESRound(V state, const V round_key) {
state = detail::SubBytes(state);
state = detail::ShiftRows(state);
state = detail::MixColumns(state);
state = Xor(state, round_key);
return state;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESLastRound(V state, const V round_key) {
state = detail::SubBytes(state);
state = detail::ShiftRows(state);
state = Xor(state, round_key);
return state;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESInvMixColumns(V state) {
return detail::InvMixColumns(state);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESRoundInv(V state, const V round_key) {
state = detail::InvSubBytes(state);
state = detail::InvShiftRows(state);
state = detail::InvMixColumns(state);
state = Xor(state, round_key);
return state;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESLastRoundInv(V state, const V round_key) {
state = detail::InvSubBytes(state);
state = detail::InvShiftRows(state);
state = Xor(state, round_key);
return state;
}
template <uint8_t kRcon, class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESKeyGenAssist(V v) {
alignas(16) static constexpr uint8_t kRconXorMask[16] = {
0, 0, 0, 0, kRcon, 0, 0, 0, 0, 0, 0, 0, kRcon, 0, 0, 0};
alignas(16) static constexpr uint8_t kRotWordShuffle[16] = {
4, 5, 6, 7, 5, 6, 7, 4, 12, 13, 14, 15, 13, 14, 15, 12};
const DFromV<decltype(v)> d;
const auto sub_word_result = detail::SubBytes(v);
const auto rot_word_result =
TableLookupBytes(sub_word_result, LoadDup128(d, kRotWordShuffle));
return Xor(rot_word_result, LoadDup128(d, kRconXorMask));
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CLMulLower(V a, V b) {
const DFromV<V> d;
static_assert(IsSame<TFromD<decltype(d)>, uint64_t>(), "V must be u64");
const auto k1 = Set(d, 0x1111111111111111ULL);
const auto k2 = Set(d, 0x2222222222222222ULL);
const auto k4 = Set(d, 0x4444444444444444ULL);
const auto k8 = Set(d, 0x8888888888888888ULL);
const auto a0 = And(a, k1);
const auto a1 = And(a, k2);
const auto a2 = And(a, k4);
const auto a3 = And(a, k8);
const auto b0 = And(b, k1);
const auto b1 = And(b, k2);
const auto b2 = And(b, k4);
const auto b3 = And(b, k8);
auto m0 = Xor(MulEven(a0, b0), MulEven(a1, b3));
auto m1 = Xor(MulEven(a0, b1), MulEven(a1, b0));
auto m2 = Xor(MulEven(a0, b2), MulEven(a1, b1));
auto m3 = Xor(MulEven(a0, b3), MulEven(a1, b2));
m0 = Xor(m0, Xor(MulEven(a2, b2), MulEven(a3, b1)));
m1 = Xor(m1, Xor(MulEven(a2, b3), MulEven(a3, b2)));
m2 = Xor(m2, Xor(MulEven(a2, b0), MulEven(a3, b3)));
m3 = Xor(m3, Xor(MulEven(a2, b1), MulEven(a3, b0)));
return Or(Or(And(m0, k1), And(m1, k2)), Or(And(m2, k4), And(m3, k8)));
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CLMulUpper(V a, V b) {
const DFromV<V> d;
static_assert(IsSame<TFromD<decltype(d)>, uint64_t>(), "V must be u64");
const auto k1 = Set(d, 0x1111111111111111ULL);
const auto k2 = Set(d, 0x2222222222222222ULL);
const auto k4 = Set(d, 0x4444444444444444ULL);
const auto k8 = Set(d, 0x8888888888888888ULL);
const auto a0 = And(a, k1);
const auto a1 = And(a, k2);
const auto a2 = And(a, k4);
const auto a3 = And(a, k8);
const auto b0 = And(b, k1);
const auto b1 = And(b, k2);
const auto b2 = And(b, k4);
const auto b3 = And(b, k8);
auto m0 = Xor(MulOdd(a0, b0), MulOdd(a1, b3));
auto m1 = Xor(MulOdd(a0, b1), MulOdd(a1, b0));
auto m2 = Xor(MulOdd(a0, b2), MulOdd(a1, b1));
auto m3 = Xor(MulOdd(a0, b3), MulOdd(a1, b2));
m0 = Xor(m0, Xor(MulOdd(a2, b2), MulOdd(a3, b1)));
m1 = Xor(m1, Xor(MulOdd(a2, b3), MulOdd(a3, b2)));
m2 = Xor(m2, Xor(MulOdd(a2, b0), MulOdd(a3, b3)));
m3 = Xor(m3, Xor(MulOdd(a2, b1), MulOdd(a3, b0)));
return Or(Or(And(m0, k1), And(m1, k2)), Or(And(m2, k4), And(m3, k8)));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr,
hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr, void* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
alignas(16) constexpr uint8_t kLookup[16] = {
0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
};
const auto lo = And(v, Set(d, uint8_t{0xF}));
const auto hi = ShiftRight<4>(v);
const auto lookup = LoadDup128(d, kLookup);
return Add(TableLookupBytes(lookup, hi), TableLookupBytes(lookup, lo));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
const V k33 = Set(d, uint8_t{0x33});
v = Sub(v, And(ShiftRight<1>(v), Set(d, uint8_t{0x55})));
v = Add(And(ShiftRight<2>(v), k33), And(v, k33));
return And(Add(v, ShiftRight<4>(v)), Set(d, uint8_t{0x0F}));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
const Repartition<uint8_t, decltype(d)> d8;
const auto vals = BitCast(d, PopulationCount(BitCast(d8, v)));
return Add(ShiftRight<8>(vals), And(vals, Set(d, uint16_t{0xFF})));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
Repartition<uint16_t, decltype(d)> d16;
auto vals = BitCast(d, PopulationCount(BitCast(d16, v)));
return Add(ShiftRight<16>(vals), And(vals, Set(d, uint32_t{0xFF})));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
Repartition<uint32_t, decltype(d)> d32;
auto vals = BitCast(d, PopulationCount(BitCast(d32, v)));
return Add(ShiftRight<32>(vals), And(vals, Set(d, 0xFFULL)));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (1)>* = nullptr,
hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16 / 2>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V operator*(const V a, const V b) {
const DFromV<decltype(a)> d;
const Rebind<MakeWide<TFromV<V>>, decltype(d)> dw;
const RebindToUnsigned<decltype(d)> du;
const RebindToUnsigned<decltype(dw)> dwu;
const VFromD<decltype(dw)> mul = PromoteTo(dw, a) * PromoteTo(dw, b);
return BitCast(d, TruncateTo(du, BitCast(dwu, mul)));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (1)>* = nullptr,
hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) > 16 / 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V operator*(const V a, const V b) {
const DFromV<decltype(a)> d;
const Half<decltype(d)> dh;
const Twice<RepartitionToWide<decltype(dh)>> dw;
const VFromD<decltype(dw)> a0 = PromoteTo(dw, LowerHalf(dh, a));
const VFromD<decltype(dw)> a1 = PromoteTo(dw, UpperHalf(dh, a));
const VFromD<decltype(dw)> b0 = PromoteTo(dw, LowerHalf(dh, b));
const VFromD<decltype(dw)> b1 = PromoteTo(dw, UpperHalf(dh, b));
const VFromD<decltype(dw)> m0 = a0 * b0;
const VFromD<decltype(dw)> m1 = a1 * b1;
return ConcatEven(d, BitCast(d, m1), BitCast(d, m0));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (8)>* = nullptr, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) == 8>* = nullptr,
hwy::EnableIf<!hwy::IsFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V operator*(V x, V y) {
const DFromV<V> d;
using T = TFromD<decltype(d)>;
using TU = MakeUnsigned<T>;
const TU xu = static_cast<TU>(GetLane(x));
const TU yu = static_cast<TU>(GetLane(y));
return Set(d, static_cast<T>(xu * yu));
}
template <class V, class D64 = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D64>, uint64_t>()>* = nullptr,
hwy::EnableIf<(D64::kPrivateLanes * sizeof(TFromD<D64>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V operator*(V x, V y) {
RepartitionToNarrow<D64> d32;
auto x32 = BitCast(d32, x);
auto y32 = BitCast(d32, y);
auto lolo = BitCast(d32, MulEven(x32, y32));
auto lohi = BitCast(d32, MulEven(x32, BitCast(d32, ShiftRight<32>(y))));
auto hilo = BitCast(d32, MulEven(BitCast(d32, ShiftRight<32>(x)), y32));
auto hi = BitCast(d32, ShiftLeft<32>(BitCast(D64{}, lohi + hilo)));
return BitCast(D64{}, lolo + hi);
}
template <class V, class DI64 = DFromV<V>, hwy::EnableIf<IsSame<TFromD<DI64>, int64_t>()>* = nullptr,
hwy::EnableIf<(DI64::kPrivateLanes * sizeof(TFromD<DI64>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V operator*(V x, V y) {
RebindToUnsigned<DI64> du64;
return BitCast(DI64{}, BitCast(du64, x) * BitCast(du64, y));
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V MulAdd(V mul, V x, V add) {
return Add(Mul(mul, x), add);
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V NegMulAdd(V mul, V x, V add) {
return Sub(add, Mul(mul, x));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ApproximateReciprocal(V v) {
const DFromV<decltype(v)> d;
return Div(Set(d, 1.0), v);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ApproximateReciprocalSqrt(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const auto half = Mul(v, Set(d, 0.5));
const auto guess = BitCast(d, Sub(Set(du, uint64_t{0x5FE6EB50C7B537A9u}),
ShiftRight<1>(BitCast(du, v))));
return Mul(guess, NegMulAdd(Mul(half, guess), guess, Set(d, 1.5)));
}
template <class V, class D, typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressBitsStore(V v, const uint8_t* __restrict__ bits, D d,
T* unaligned) {
alignas(16) T lanes[MaxLanes(d)];
Store(v, d, lanes);
const Simd<T, ((MaxLanes(d)) < (8) ? (MaxLanes(d)) : (8)), 0> d8;
T* __restrict__ pos = unaligned;
alignas(16) constexpr T table[2048] = {
0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
1, 0, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
2, 0, 1, 3, 4, 5, 6, 7, 0, 2, 1, 3, 4, 5, 6, 7,
1, 2, 0, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
3, 0, 1, 2, 4, 5, 6, 7, 0, 3, 1, 2, 4, 5, 6, 7,
1, 3, 0, 2, 4, 5, 6, 7, 0, 1, 3, 2, 4, 5, 6, 7,
2, 3, 0, 1, 4, 5, 6, 7, 0, 2, 3, 1, 4, 5, 6, 7,
1, 2, 3, 0, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
4, 0, 1, 2, 3, 5, 6, 7, 0, 4, 1, 2, 3, 5, 6, 7,
1, 4, 0, 2, 3, 5, 6, 7, 0, 1, 4, 2, 3, 5, 6, 7,
2, 4, 0, 1, 3, 5, 6, 7, 0, 2, 4, 1, 3, 5, 6, 7,
1, 2, 4, 0, 3, 5, 6, 7, 0, 1, 2, 4, 3, 5, 6, 7,
3, 4, 0, 1, 2, 5, 6, 7, 0, 3, 4, 1, 2, 5, 6, 7,
1, 3, 4, 0, 2, 5, 6, 7, 0, 1, 3, 4, 2, 5, 6, 7,
2, 3, 4, 0, 1, 5, 6, 7, 0, 2, 3, 4, 1, 5, 6, 7,
1, 2, 3, 4, 0, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
5, 0, 1, 2, 3, 4, 6, 7, 0, 5, 1, 2, 3, 4, 6, 7,
1, 5, 0, 2, 3, 4, 6, 7, 0, 1, 5, 2, 3, 4, 6, 7,
2, 5, 0, 1, 3, 4, 6, 7, 0, 2, 5, 1, 3, 4, 6, 7,
1, 2, 5, 0, 3, 4, 6, 7, 0, 1, 2, 5, 3, 4, 6, 7,
3, 5, 0, 1, 2, 4, 6, 7, 0, 3, 5, 1, 2, 4, 6, 7,
1, 3, 5, 0, 2, 4, 6, 7, 0, 1, 3, 5, 2, 4, 6, 7,
2, 3, 5, 0, 1, 4, 6, 7, 0, 2, 3, 5, 1, 4, 6, 7,
1, 2, 3, 5, 0, 4, 6, 7, 0, 1, 2, 3, 5, 4, 6, 7,
4, 5, 0, 1, 2, 3, 6, 7, 0, 4, 5, 1, 2, 3, 6, 7,
1, 4, 5, 0, 2, 3, 6, 7, 0, 1, 4, 5, 2, 3, 6, 7,
2, 4, 5, 0, 1, 3, 6, 7, 0, 2, 4, 5, 1, 3, 6, 7,
1, 2, 4, 5, 0, 3, 6, 7, 0, 1, 2, 4, 5, 3, 6, 7,
3, 4, 5, 0, 1, 2, 6, 7, 0, 3, 4, 5, 1, 2, 6, 7,
1, 3, 4, 5, 0, 2, 6, 7, 0, 1, 3, 4, 5, 2, 6, 7,
2, 3, 4, 5, 0, 1, 6, 7, 0, 2, 3, 4, 5, 1, 6, 7,
1, 2, 3, 4, 5, 0, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
6, 0, 1, 2, 3, 4, 5, 7, 0, 6, 1, 2, 3, 4, 5, 7,
1, 6, 0, 2, 3, 4, 5, 7, 0, 1, 6, 2, 3, 4, 5, 7,
2, 6, 0, 1, 3, 4, 5, 7, 0, 2, 6, 1, 3, 4, 5, 7,
1, 2, 6, 0, 3, 4, 5, 7, 0, 1, 2, 6, 3, 4, 5, 7,
3, 6, 0, 1, 2, 4, 5, 7, 0, 3, 6, 1, 2, 4, 5, 7,
1, 3, 6, 0, 2, 4, 5, 7, 0, 1, 3, 6, 2, 4, 5, 7,
2, 3, 6, 0, 1, 4, 5, 7, 0, 2, 3, 6, 1, 4, 5, 7,
1, 2, 3, 6, 0, 4, 5, 7, 0, 1, 2, 3, 6, 4, 5, 7,
4, 6, 0, 1, 2, 3, 5, 7, 0, 4, 6, 1, 2, 3, 5, 7,
1, 4, 6, 0, 2, 3, 5, 7, 0, 1, 4, 6, 2, 3, 5, 7,
2, 4, 6, 0, 1, 3, 5, 7, 0, 2, 4, 6, 1, 3, 5, 7,
1, 2, 4, 6, 0, 3, 5, 7, 0, 1, 2, 4, 6, 3, 5, 7,
3, 4, 6, 0, 1, 2, 5, 7, 0, 3, 4, 6, 1, 2, 5, 7,
1, 3, 4, 6, 0, 2, 5, 7, 0, 1, 3, 4, 6, 2, 5, 7,
2, 3, 4, 6, 0, 1, 5, 7, 0, 2, 3, 4, 6, 1, 5, 7,
1, 2, 3, 4, 6, 0, 5, 7, 0, 1, 2, 3, 4, 6, 5, 7,
5, 6, 0, 1, 2, 3, 4, 7, 0, 5, 6, 1, 2, 3, 4, 7,
1, 5, 6, 0, 2, 3, 4, 7, 0, 1, 5, 6, 2, 3, 4, 7,
2, 5, 6, 0, 1, 3, 4, 7, 0, 2, 5, 6, 1, 3, 4, 7,
1, 2, 5, 6, 0, 3, 4, 7, 0, 1, 2, 5, 6, 3, 4, 7,
3, 5, 6, 0, 1, 2, 4, 7, 0, 3, 5, 6, 1, 2, 4, 7,
1, 3, 5, 6, 0, 2, 4, 7, 0, 1, 3, 5, 6, 2, 4, 7,
2, 3, 5, 6, 0, 1, 4, 7, 0, 2, 3, 5, 6, 1, 4, 7,
1, 2, 3, 5, 6, 0, 4, 7, 0, 1, 2, 3, 5, 6, 4, 7,
4, 5, 6, 0, 1, 2, 3, 7, 0, 4, 5, 6, 1, 2, 3, 7,
1, 4, 5, 6, 0, 2, 3, 7, 0, 1, 4, 5, 6, 2, 3, 7,
2, 4, 5, 6, 0, 1, 3, 7, 0, 2, 4, 5, 6, 1, 3, 7,
1, 2, 4, 5, 6, 0, 3, 7, 0, 1, 2, 4, 5, 6, 3, 7,
3, 4, 5, 6, 0, 1, 2, 7, 0, 3, 4, 5, 6, 1, 2, 7,
1, 3, 4, 5, 6, 0, 2, 7, 0, 1, 3, 4, 5, 6, 2, 7,
2, 3, 4, 5, 6, 0, 1, 7, 0, 2, 3, 4, 5, 6, 1, 7,
1, 2, 3, 4, 5, 6, 0, 7, 0, 1, 2, 3, 4, 5, 6, 7,
7, 0, 1, 2, 3, 4, 5, 6, 0, 7, 1, 2, 3, 4, 5, 6,
1, 7, 0, 2, 3, 4, 5, 6, 0, 1, 7, 2, 3, 4, 5, 6,
2, 7, 0, 1, 3, 4, 5, 6, 0, 2, 7, 1, 3, 4, 5, 6,
1, 2, 7, 0, 3, 4, 5, 6, 0, 1, 2, 7, 3, 4, 5, 6,
3, 7, 0, 1, 2, 4, 5, 6, 0, 3, 7, 1, 2, 4, 5, 6,
1, 3, 7, 0, 2, 4, 5, 6, 0, 1, 3, 7, 2, 4, 5, 6,
2, 3, 7, 0, 1, 4, 5, 6, 0, 2, 3, 7, 1, 4, 5, 6,
1, 2, 3, 7, 0, 4, 5, 6, 0, 1, 2, 3, 7, 4, 5, 6,
4, 7, 0, 1, 2, 3, 5, 6, 0, 4, 7, 1, 2, 3, 5, 6,
1, 4, 7, 0, 2, 3, 5, 6, 0, 1, 4, 7, 2, 3, 5, 6,
2, 4, 7, 0, 1, 3, 5, 6, 0, 2, 4, 7, 1, 3, 5, 6,
1, 2, 4, 7, 0, 3, 5, 6, 0, 1, 2, 4, 7, 3, 5, 6,
3, 4, 7, 0, 1, 2, 5, 6, 0, 3, 4, 7, 1, 2, 5, 6,
1, 3, 4, 7, 0, 2, 5, 6, 0, 1, 3, 4, 7, 2, 5, 6,
2, 3, 4, 7, 0, 1, 5, 6, 0, 2, 3, 4, 7, 1, 5, 6,
1, 2, 3, 4, 7, 0, 5, 6, 0, 1, 2, 3, 4, 7, 5, 6,
5, 7, 0, 1, 2, 3, 4, 6, 0, 5, 7, 1, 2, 3, 4, 6,
1, 5, 7, 0, 2, 3, 4, 6, 0, 1, 5, 7, 2, 3, 4, 6,
2, 5, 7, 0, 1, 3, 4, 6, 0, 2, 5, 7, 1, 3, 4, 6,
1, 2, 5, 7, 0, 3, 4, 6, 0, 1, 2, 5, 7, 3, 4, 6,
3, 5, 7, 0, 1, 2, 4, 6, 0, 3, 5, 7, 1, 2, 4, 6,
1, 3, 5, 7, 0, 2, 4, 6, 0, 1, 3, 5, 7, 2, 4, 6,
2, 3, 5, 7, 0, 1, 4, 6, 0, 2, 3, 5, 7, 1, 4, 6,
1, 2, 3, 5, 7, 0, 4, 6, 0, 1, 2, 3, 5, 7, 4, 6,
4, 5, 7, 0, 1, 2, 3, 6, 0, 4, 5, 7, 1, 2, 3, 6,
1, 4, 5, 7, 0, 2, 3, 6, 0, 1, 4, 5, 7, 2, 3, 6,
2, 4, 5, 7, 0, 1, 3, 6, 0, 2, 4, 5, 7, 1, 3, 6,
1, 2, 4, 5, 7, 0, 3, 6, 0, 1, 2, 4, 5, 7, 3, 6,
3, 4, 5, 7, 0, 1, 2, 6, 0, 3, 4, 5, 7, 1, 2, 6,
1, 3, 4, 5, 7, 0, 2, 6, 0, 1, 3, 4, 5, 7, 2, 6,
2, 3, 4, 5, 7, 0, 1, 6, 0, 2, 3, 4, 5, 7, 1, 6,
1, 2, 3, 4, 5, 7, 0, 6, 0, 1, 2, 3, 4, 5, 7, 6,
6, 7, 0, 1, 2, 3, 4, 5, 0, 6, 7, 1, 2, 3, 4, 5,
1, 6, 7, 0, 2, 3, 4, 5, 0, 1, 6, 7, 2, 3, 4, 5,
2, 6, 7, 0, 1, 3, 4, 5, 0, 2, 6, 7, 1, 3, 4, 5,
1, 2, 6, 7, 0, 3, 4, 5, 0, 1, 2, 6, 7, 3, 4, 5,
3, 6, 7, 0, 1, 2, 4, 5, 0, 3, 6, 7, 1, 2, 4, 5,
1, 3, 6, 7, 0, 2, 4, 5, 0, 1, 3, 6, 7, 2, 4, 5,
2, 3, 6, 7, 0, 1, 4, 5, 0, 2, 3, 6, 7, 1, 4, 5,
1, 2, 3, 6, 7, 0, 4, 5, 0, 1, 2, 3, 6, 7, 4, 5,
4, 6, 7, 0, 1, 2, 3, 5, 0, 4, 6, 7, 1, 2, 3, 5,
1, 4, 6, 7, 0, 2, 3, 5, 0, 1, 4, 6, 7, 2, 3, 5,
2, 4, 6, 7, 0, 1, 3, 5, 0, 2, 4, 6, 7, 1, 3, 5,
1, 2, 4, 6, 7, 0, 3, 5, 0, 1, 2, 4, 6, 7, 3, 5,
3, 4, 6, 7, 0, 1, 2, 5, 0, 3, 4, 6, 7, 1, 2, 5,
1, 3, 4, 6, 7, 0, 2, 5, 0, 1, 3, 4, 6, 7, 2, 5,
2, 3, 4, 6, 7, 0, 1, 5, 0, 2, 3, 4, 6, 7, 1, 5,
1, 2, 3, 4, 6, 7, 0, 5, 0, 1, 2, 3, 4, 6, 7, 5,
5, 6, 7, 0, 1, 2, 3, 4, 0, 5, 6, 7, 1, 2, 3, 4,
1, 5, 6, 7, 0, 2, 3, 4, 0, 1, 5, 6, 7, 2, 3, 4,
2, 5, 6, 7, 0, 1, 3, 4, 0, 2, 5, 6, 7, 1, 3, 4,
1, 2, 5, 6, 7, 0, 3, 4, 0, 1, 2, 5, 6, 7, 3, 4,
3, 5, 6, 7, 0, 1, 2, 4, 0, 3, 5, 6, 7, 1, 2, 4,
1, 3, 5, 6, 7, 0, 2, 4, 0, 1, 3, 5, 6, 7, 2, 4,
2, 3, 5, 6, 7, 0, 1, 4, 0, 2, 3, 5, 6, 7, 1, 4,
1, 2, 3, 5, 6, 7, 0, 4, 0, 1, 2, 3, 5, 6, 7, 4,
4, 5, 6, 7, 0, 1, 2, 3, 0, 4, 5, 6, 7, 1, 2, 3,
1, 4, 5, 6, 7, 0, 2, 3, 0, 1, 4, 5, 6, 7, 2, 3,
2, 4, 5, 6, 7, 0, 1, 3, 0, 2, 4, 5, 6, 7, 1, 3,
1, 2, 4, 5, 6, 7, 0, 3, 0, 1, 2, 4, 5, 6, 7, 3,
3, 4, 5, 6, 7, 0, 1, 2, 0, 3, 4, 5, 6, 7, 1, 2,
1, 3, 4, 5, 6, 7, 0, 2, 0, 1, 3, 4, 5, 6, 7, 2,
2, 3, 4, 5, 6, 7, 0, 1, 0, 2, 3, 4, 5, 6, 7, 1,
1, 2, 3, 4, 5, 6, 7, 0, 0, 1, 2, 3, 4, 5, 6, 7};
for (size_t i = 0; i < Lanes(d); i += 8) {
const size_t bits8 = bits[i / 8];
const auto indices = Load(d8, table + bits8 * 8);
const auto compressed = TableLookupBytes(LoadU(d8, lanes + i), indices);
StoreU(compressed, d8, pos);
pos += PopCount(bits8);
}
return static_cast<size_t>(pos - unaligned);
}
template <class V, class M, class D, typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressStore(V v, M mask, D d, T* __restrict__ unaligned) {
uint8_t bits[((size_t{8}) > (MaxLanes(d) / 8) ? (size_t{8}) : (MaxLanes(d) / 8))];
(void)StoreMaskBits(d, mask, bits);
return CompressBitsStore(v, bits, d, unaligned);
}
template <class V, class M, class D, typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressBlendedStore(V v, M mask, D d,
T* __restrict__ unaligned) {
alignas(16) T buf[MaxLanes(d)];
const size_t bytes = CompressStore(v, mask, d, buf);
BlendedStore(Load(d, buf), FirstN(d, bytes), d, unaligned);
return bytes;
}
template <class V, class M, typename T = TFromV<V>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Compress(V v, const M mask) {
const DFromV<V> d;
alignas(16) T lanes[MaxLanes(d)];
(void)CompressStore(v, mask, d, lanes);
return Load(d, lanes);
}
template <class V, typename T = TFromV<V>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CompressBits(V v, const uint8_t* __restrict__ bits) {
const DFromV<V> d;
alignas(16) T lanes[MaxLanes(d)];
(void)CompressBitsStore(v, bits, d, lanes);
return Load(d, lanes);
}
template <class V, class M, typename T = TFromV<V>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CompressNot(V v, M mask) {
return Compress(v, Not(mask));
}
namespace detail {
template <size_t N>
inline __attribute__((always_inline)) Vec128<uint8_t, N> IndicesForExpandFromBits(uint64_t mask_bits) {
static_assert(N <= 8, "Should only be called for half-vectors");
const Simd<uint8_t, N, 0> du8;
do { } while (0);
alignas(16) static constexpr uint8_t table[2048] = {
128, 128, 128, 128, 128, 128, 128, 128,
0, 128, 128, 128, 128, 128, 128, 128,
128, 0, 128, 128, 128, 128, 128, 128,
0, 1, 128, 128, 128, 128, 128, 128,
128, 128, 0, 128, 128, 128, 128, 128,
0, 128, 1, 128, 128, 128, 128, 128,
128, 0, 1, 128, 128, 128, 128, 128,
0, 1, 2, 128, 128, 128, 128, 128,
128, 128, 128, 0, 128, 128, 128, 128,
0, 128, 128, 1, 128, 128, 128, 128,
128, 0, 128, 1, 128, 128, 128, 128,
0, 1, 128, 2, 128, 128, 128, 128,
128, 128, 0, 1, 128, 128, 128, 128,
0, 128, 1, 2, 128, 128, 128, 128,
128, 0, 1, 2, 128, 128, 128, 128,
0, 1, 2, 3, 128, 128, 128, 128,
128, 128, 128, 128, 0, 128, 128, 128,
0, 128, 128, 128, 1, 128, 128, 128,
128, 0, 128, 128, 1, 128, 128, 128,
0, 1, 128, 128, 2, 128, 128, 128,
128, 128, 0, 128, 1, 128, 128, 128,
0, 128, 1, 128, 2, 128, 128, 128,
128, 0, 1, 128, 2, 128, 128, 128,
0, 1, 2, 128, 3, 128, 128, 128,
128, 128, 128, 0, 1, 128, 128, 128,
0, 128, 128, 1, 2, 128, 128, 128,
128, 0, 128, 1, 2, 128, 128, 128,
0, 1, 128, 2, 3, 128, 128, 128,
128, 128, 0, 1, 2, 128, 128, 128,
0, 128, 1, 2, 3, 128, 128, 128,
128, 0, 1, 2, 3, 128, 128, 128,
0, 1, 2, 3, 4, 128, 128, 128,
128, 128, 128, 128, 128, 0, 128, 128,
0, 128, 128, 128, 128, 1, 128, 128,
128, 0, 128, 128, 128, 1, 128, 128,
0, 1, 128, 128, 128, 2, 128, 128,
128, 128, 0, 128, 128, 1, 128, 128,
0, 128, 1, 128, 128, 2, 128, 128,
128, 0, 1, 128, 128, 2, 128, 128,
0, 1, 2, 128, 128, 3, 128, 128,
128, 128, 128, 0, 128, 1, 128, 128,
0, 128, 128, 1, 128, 2, 128, 128,
128, 0, 128, 1, 128, 2, 128, 128,
0, 1, 128, 2, 128, 3, 128, 128,
128, 128, 0, 1, 128, 2, 128, 128,
0, 128, 1, 2, 128, 3, 128, 128,
128, 0, 1, 2, 128, 3, 128, 128,
0, 1, 2, 3, 128, 4, 128, 128,
128, 128, 128, 128, 0, 1, 128, 128,
0, 128, 128, 128, 1, 2, 128, 128,
128, 0, 128, 128, 1, 2, 128, 128,
0, 1, 128, 128, 2, 3, 128, 128,
128, 128, 0, 128, 1, 2, 128, 128,
0, 128, 1, 128, 2, 3, 128, 128,
128, 0, 1, 128, 2, 3, 128, 128,
0, 1, 2, 128, 3, 4, 128, 128,
128, 128, 128, 0, 1, 2, 128, 128,
0, 128, 128, 1, 2, 3, 128, 128,
128, 0, 128, 1, 2, 3, 128, 128,
0, 1, 128, 2, 3, 4, 128, 128,
128, 128, 0, 1, 2, 3, 128, 128,
0, 128, 1, 2, 3, 4, 128, 128,
128, 0, 1, 2, 3, 4, 128, 128,
0, 1, 2, 3, 4, 5, 128, 128,
128, 128, 128, 128, 128, 128, 0, 128,
0, 128, 128, 128, 128, 128, 1, 128,
128, 0, 128, 128, 128, 128, 1, 128,
0, 1, 128, 128, 128, 128, 2, 128,
128, 128, 0, 128, 128, 128, 1, 128,
0, 128, 1, 128, 128, 128, 2, 128,
128, 0, 1, 128, 128, 128, 2, 128,
0, 1, 2, 128, 128, 128, 3, 128,
128, 128, 128, 0, 128, 128, 1, 128,
0, 128, 128, 1, 128, 128, 2, 128,
128, 0, 128, 1, 128, 128, 2, 128,
0, 1, 128, 2, 128, 128, 3, 128,
128, 128, 0, 1, 128, 128, 2, 128,
0, 128, 1, 2, 128, 128, 3, 128,
128, 0, 1, 2, 128, 128, 3, 128,
0, 1, 2, 3, 128, 128, 4, 128,
128, 128, 128, 128, 0, 128, 1, 128,
0, 128, 128, 128, 1, 128, 2, 128,
128, 0, 128, 128, 1, 128, 2, 128,
0, 1, 128, 128, 2, 128, 3, 128,
128, 128, 0, 128, 1, 128, 2, 128,
0, 128, 1, 128, 2, 128, 3, 128,
128, 0, 1, 128, 2, 128, 3, 128,
0, 1, 2, 128, 3, 128, 4, 128,
128, 128, 128, 0, 1, 128, 2, 128,
0, 128, 128, 1, 2, 128, 3, 128,
128, 0, 128, 1, 2, 128, 3, 128,
0, 1, 128, 2, 3, 128, 4, 128,
128, 128, 0, 1, 2, 128, 3, 128,
0, 128, 1, 2, 3, 128, 4, 128,
128, 0, 1, 2, 3, 128, 4, 128,
0, 1, 2, 3, 4, 128, 5, 128,
128, 128, 128, 128, 128, 0, 1, 128,
0, 128, 128, 128, 128, 1, 2, 128,
128, 0, 128, 128, 128, 1, 2, 128,
0, 1, 128, 128, 128, 2, 3, 128,
128, 128, 0, 128, 128, 1, 2, 128,
0, 128, 1, 128, 128, 2, 3, 128,
128, 0, 1, 128, 128, 2, 3, 128,
0, 1, 2, 128, 128, 3, 4, 128,
128, 128, 128, 0, 128, 1, 2, 128,
0, 128, 128, 1, 128, 2, 3, 128,
128, 0, 128, 1, 128, 2, 3, 128,
0, 1, 128, 2, 128, 3, 4, 128,
128, 128, 0, 1, 128, 2, 3, 128,
0, 128, 1, 2, 128, 3, 4, 128,
128, 0, 1, 2, 128, 3, 4, 128,
0, 1, 2, 3, 128, 4, 5, 128,
128, 128, 128, 128, 0, 1, 2, 128,
0, 128, 128, 128, 1, 2, 3, 128,
128, 0, 128, 128, 1, 2, 3, 128,
0, 1, 128, 128, 2, 3, 4, 128,
128, 128, 0, 128, 1, 2, 3, 128,
0, 128, 1, 128, 2, 3, 4, 128,
128, 0, 1, 128, 2, 3, 4, 128,
0, 1, 2, 128, 3, 4, 5, 128,
128, 128, 128, 0, 1, 2, 3, 128,
0, 128, 128, 1, 2, 3, 4, 128,
128, 0, 128, 1, 2, 3, 4, 128,
0, 1, 128, 2, 3, 4, 5, 128,
128, 128, 0, 1, 2, 3, 4, 128,
0, 128, 1, 2, 3, 4, 5, 128,
128, 0, 1, 2, 3, 4, 5, 128,
0, 1, 2, 3, 4, 5, 6, 128,
128, 128, 128, 128, 128, 128, 128, 0,
0, 128, 128, 128, 128, 128, 128, 1,
128, 0, 128, 128, 128, 128, 128, 1,
0, 1, 128, 128, 128, 128, 128, 2,
128, 128, 0, 128, 128, 128, 128, 1,
0, 128, 1, 128, 128, 128, 128, 2,
128, 0, 1, 128, 128, 128, 128, 2,
0, 1, 2, 128, 128, 128, 128, 3,
128, 128, 128, 0, 128, 128, 128, 1,
0, 128, 128, 1, 128, 128, 128, 2,
128, 0, 128, 1, 128, 128, 128, 2,
0, 1, 128, 2, 128, 128, 128, 3,
128, 128, 0, 1, 128, 128, 128, 2,
0, 128, 1, 2, 128, 128, 128, 3,
128, 0, 1, 2, 128, 128, 128, 3,
0, 1, 2, 3, 128, 128, 128, 4,
128, 128, 128, 128, 0, 128, 128, 1,
0, 128, 128, 128, 1, 128, 128, 2,
128, 0, 128, 128, 1, 128, 128, 2,
0, 1, 128, 128, 2, 128, 128, 3,
128, 128, 0, 128, 1, 128, 128, 2,
0, 128, 1, 128, 2, 128, 128, 3,
128, 0, 1, 128, 2, 128, 128, 3,
0, 1, 2, 128, 3, 128, 128, 4,
128, 128, 128, 0, 1, 128, 128, 2,
0, 128, 128, 1, 2, 128, 128, 3,
128, 0, 128, 1, 2, 128, 128, 3,
0, 1, 128, 2, 3, 128, 128, 4,
128, 128, 0, 1, 2, 128, 128, 3,
0, 128, 1, 2, 3, 128, 128, 4,
128, 0, 1, 2, 3, 128, 128, 4,
0, 1, 2, 3, 4, 128, 128, 5,
128, 128, 128, 128, 128, 0, 128, 1,
0, 128, 128, 128, 128, 1, 128, 2,
128, 0, 128, 128, 128, 1, 128, 2,
0, 1, 128, 128, 128, 2, 128, 3,
128, 128, 0, 128, 128, 1, 128, 2,
0, 128, 1, 128, 128, 2, 128, 3,
128, 0, 1, 128, 128, 2, 128, 3,
0, 1, 2, 128, 128, 3, 128, 4,
128, 128, 128, 0, 128, 1, 128, 2,
0, 128, 128, 1, 128, 2, 128, 3,
128, 0, 128, 1, 128, 2, 128, 3,
0, 1, 128, 2, 128, 3, 128, 4,
128, 128, 0, 1, 128, 2, 128, 3,
0, 128, 1, 2, 128, 3, 128, 4,
128, 0, 1, 2, 128, 3, 128, 4,
0, 1, 2, 3, 128, 4, 128, 5,
128, 128, 128, 128, 0, 1, 128, 2,
0, 128, 128, 128, 1, 2, 128, 3,
128, 0, 128, 128, 1, 2, 128, 3,
0, 1, 128, 128, 2, 3, 128, 4,
128, 128, 0, 128, 1, 2, 128, 3,
0, 128, 1, 128, 2, 3, 128, 4,
128, 0, 1, 128, 2, 3, 128, 4,
0, 1, 2, 128, 3, 4, 128, 5,
128, 128, 128, 0, 1, 2, 128, 3,
0, 128, 128, 1, 2, 3, 128, 4,
128, 0, 128, 1, 2, 3, 128, 4,
0, 1, 128, 2, 3, 4, 128, 5,
128, 128, 0, 1, 2, 3, 128, 4,
0, 128, 1, 2, 3, 4, 128, 5,
128, 0, 1, 2, 3, 4, 128, 5,
0, 1, 2, 3, 4, 5, 128, 6,
128, 128, 128, 128, 128, 128, 0, 1,
0, 128, 128, 128, 128, 128, 1, 2,
128, 0, 128, 128, 128, 128, 1, 2,
0, 1, 128, 128, 128, 128, 2, 3,
128, 128, 0, 128, 128, 128, 1, 2,
0, 128, 1, 128, 128, 128, 2, 3,
128, 0, 1, 128, 128, 128, 2, 3,
0, 1, 2, 128, 128, 128, 3, 4,
128, 128, 128, 0, 128, 128, 1, 2,
0, 128, 128, 1, 128, 128, 2, 3,
128, 0, 128, 1, 128, 128, 2, 3,
0, 1, 128, 2, 128, 128, 3, 4,
128, 128, 0, 1, 128, 128, 2, 3,
0, 128, 1, 2, 128, 128, 3, 4,
128, 0, 1, 2, 128, 128, 3, 4,
0, 1, 2, 3, 128, 128, 4, 5,
128, 128, 128, 128, 0, 128, 1, 2,
0, 128, 128, 128, 1, 128, 2, 3,
128, 0, 128, 128, 1, 128, 2, 3,
0, 1, 128, 128, 2, 128, 3, 4,
128, 128, 0, 128, 1, 128, 2, 3,
0, 128, 1, 128, 2, 128, 3, 4,
128, 0, 1, 128, 2, 128, 3, 4,
0, 1, 2, 128, 3, 128, 4, 5,
128, 128, 128, 0, 1, 128, 2, 3,
0, 128, 128, 1, 2, 128, 3, 4,
128, 0, 128, 1, 2, 128, 3, 4,
0, 1, 128, 2, 3, 128, 4, 5,
128, 128, 0, 1, 2, 128, 3, 4,
0, 128, 1, 2, 3, 128, 4, 5,
128, 0, 1, 2, 3, 128, 4, 5,
0, 1, 2, 3, 4, 128, 5, 6,
128, 128, 128, 128, 128, 0, 1, 2,
0, 128, 128, 128, 128, 1, 2, 3,
128, 0, 128, 128, 128, 1, 2, 3,
0, 1, 128, 128, 128, 2, 3, 4,
128, 128, 0, 128, 128, 1, 2, 3,
0, 128, 1, 128, 128, 2, 3, 4,
128, 0, 1, 128, 128, 2, 3, 4,
0, 1, 2, 128, 128, 3, 4, 5,
128, 128, 128, 0, 128, 1, 2, 3,
0, 128, 128, 1, 128, 2, 3, 4,
128, 0, 128, 1, 128, 2, 3, 4,
0, 1, 128, 2, 128, 3, 4, 5,
128, 128, 0, 1, 128, 2, 3, 4,
0, 128, 1, 2, 128, 3, 4, 5,
128, 0, 1, 2, 128, 3, 4, 5,
0, 1, 2, 3, 128, 4, 5, 6,
128, 128, 128, 128, 0, 1, 2, 3,
0, 128, 128, 128, 1, 2, 3, 4,
128, 0, 128, 128, 1, 2, 3, 4,
0, 1, 128, 128, 2, 3, 4, 5,
128, 128, 0, 128, 1, 2, 3, 4,
0, 128, 1, 128, 2, 3, 4, 5,
128, 0, 1, 128, 2, 3, 4, 5,
0, 1, 2, 128, 3, 4, 5, 6,
128, 128, 128, 0, 1, 2, 3, 4,
0, 128, 128, 1, 2, 3, 4, 5,
128, 0, 128, 1, 2, 3, 4, 5,
0, 1, 128, 2, 3, 4, 5, 6,
128, 128, 0, 1, 2, 3, 4, 5,
0, 128, 1, 2, 3, 4, 5, 6,
128, 0, 1, 2, 3, 4, 5, 6,
0, 1, 2, 3, 4, 5, 6, 7};
return LoadU(du8, table + mask_bits * 8);
}
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr, hwy::EnableIf<N * sizeof(T) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Expand(Vec128<T, N> v, Mask128<T, N> mask) {
const DFromV<decltype(v)> d;
const uint64_t mask_bits = detail::BitsFromMask(mask);
const Vec128<uint8_t, N> indices =
detail::IndicesForExpandFromBits<N>(mask_bits);
return BitCast(d, TableLookupBytesOr0(v, indices));
}
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Expand(Vec128<T> v, Mask128<T> mask) {
const Full128<T> d;
const RebindToUnsigned<decltype(d)> du;
const Half<decltype(du)> duh;
const Vec128<uint8_t> vu = BitCast(du, v);
const uint64_t mask_bits = detail::BitsFromMask(mask);
const uint64_t maskL = mask_bits & 0xFF;
const uint64_t maskH = mask_bits >> 8;
alignas(16) static constexpr uint8_t iota[32] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128, 128, 128};
const VFromD<decltype(du)> shift = LoadU(du, iota + PopCount(maskL));
const VFromD<decltype(duh)> vL = LowerHalf(duh, vu);
const VFromD<decltype(duh)> vH =
LowerHalf(duh, TableLookupBytesOr0(vu, shift));
const VFromD<decltype(duh)> idxL = detail::IndicesForExpandFromBits<8>(maskL);
const VFromD<decltype(duh)> idxH = detail::IndicesForExpandFromBits<8>(maskH);
const VFromD<decltype(duh)> expandL = TableLookupBytesOr0(vL, idxL);
const VFromD<decltype(duh)> expandH = TableLookupBytesOr0(vH, idxH);
return BitCast(d, Combine(du, expandH, expandL));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Expand(Vec128<T, N> v, Mask128<T, N> mask) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const Rebind<uint8_t, decltype(d)> du8;
const uint64_t mask_bits = detail::BitsFromMask(mask);
alignas(16) static constexpr uint8_t table[2048] = {
128, 128, 128, 128, 128, 128, 128, 128,
0, 128, 128, 128, 128, 128, 128, 128,
128, 0, 128, 128, 128, 128, 128, 128,
0, 2, 128, 128, 128, 128, 128, 128,
128, 128, 0, 128, 128, 128, 128, 128,
0, 128, 2, 128, 128, 128, 128, 128,
128, 0, 2, 128, 128, 128, 128, 128,
0, 2, 4, 128, 128, 128, 128, 128,
128, 128, 128, 0, 128, 128, 128, 128,
0, 128, 128, 2, 128, 128, 128, 128,
128, 0, 128, 2, 128, 128, 128, 128,
0, 2, 128, 4, 128, 128, 128, 128,
128, 128, 0, 2, 128, 128, 128, 128,
0, 128, 2, 4, 128, 128, 128, 128,
128, 0, 2, 4, 128, 128, 128, 128,
0, 2, 4, 6, 128, 128, 128, 128,
128, 128, 128, 128, 0, 128, 128, 128,
0, 128, 128, 128, 2, 128, 128, 128,
128, 0, 128, 128, 2, 128, 128, 128,
0, 2, 128, 128, 4, 128, 128, 128,
128, 128, 0, 128, 2, 128, 128, 128,
0, 128, 2, 128, 4, 128, 128, 128,
128, 0, 2, 128, 4, 128, 128, 128,
0, 2, 4, 128, 6, 128, 128, 128,
128, 128, 128, 0, 2, 128, 128, 128,
0, 128, 128, 2, 4, 128, 128, 128,
128, 0, 128, 2, 4, 128, 128, 128,
0, 2, 128, 4, 6, 128, 128, 128,
128, 128, 0, 2, 4, 128, 128, 128,
0, 128, 2, 4, 6, 128, 128, 128,
128, 0, 2, 4, 6, 128, 128, 128,
0, 2, 4, 6, 8, 128, 128, 128,
128, 128, 128, 128, 128, 0, 128, 128,
0, 128, 128, 128, 128, 2, 128, 128,
128, 0, 128, 128, 128, 2, 128, 128,
0, 2, 128, 128, 128, 4, 128, 128,
128, 128, 0, 128, 128, 2, 128, 128,
0, 128, 2, 128, 128, 4, 128, 128,
128, 0, 2, 128, 128, 4, 128, 128,
0, 2, 4, 128, 128, 6, 128, 128,
128, 128, 128, 0, 128, 2, 128, 128,
0, 128, 128, 2, 128, 4, 128, 128,
128, 0, 128, 2, 128, 4, 128, 128,
0, 2, 128, 4, 128, 6, 128, 128,
128, 128, 0, 2, 128, 4, 128, 128,
0, 128, 2, 4, 128, 6, 128, 128,
128, 0, 2, 4, 128, 6, 128, 128,
0, 2, 4, 6, 128, 8, 128, 128,
128, 128, 128, 128, 0, 2, 128, 128,
0, 128, 128, 128, 2, 4, 128, 128,
128, 0, 128, 128, 2, 4, 128, 128,
0, 2, 128, 128, 4, 6, 128, 128,
128, 128, 0, 128, 2, 4, 128, 128,
0, 128, 2, 128, 4, 6, 128, 128,
128, 0, 2, 128, 4, 6, 128, 128,
0, 2, 4, 128, 6, 8, 128, 128,
128, 128, 128, 0, 2, 4, 128, 128,
0, 128, 128, 2, 4, 6, 128, 128,
128, 0, 128, 2, 4, 6, 128, 128,
0, 2, 128, 4, 6, 8, 128, 128,
128, 128, 0, 2, 4, 6, 128, 128,
0, 128, 2, 4, 6, 8, 128, 128,
128, 0, 2, 4, 6, 8, 128, 128,
0, 2, 4, 6, 8, 10, 128, 128,
128, 128, 128, 128, 128, 128, 0, 128,
0, 128, 128, 128, 128, 128, 2, 128,
128, 0, 128, 128, 128, 128, 2, 128,
0, 2, 128, 128, 128, 128, 4, 128,
128, 128, 0, 128, 128, 128, 2, 128,
0, 128, 2, 128, 128, 128, 4, 128,
128, 0, 2, 128, 128, 128, 4, 128,
0, 2, 4, 128, 128, 128, 6, 128,
128, 128, 128, 0, 128, 128, 2, 128,
0, 128, 128, 2, 128, 128, 4, 128,
128, 0, 128, 2, 128, 128, 4, 128,
0, 2, 128, 4, 128, 128, 6, 128,
128, 128, 0, 2, 128, 128, 4, 128,
0, 128, 2, 4, 128, 128, 6, 128,
128, 0, 2, 4, 128, 128, 6, 128,
0, 2, 4, 6, 128, 128, 8, 128,
128, 128, 128, 128, 0, 128, 2, 128,
0, 128, 128, 128, 2, 128, 4, 128,
128, 0, 128, 128, 2, 128, 4, 128,
0, 2, 128, 128, 4, 128, 6, 128,
128, 128, 0, 128, 2, 128, 4, 128,
0, 128, 2, 128, 4, 128, 6, 128,
128, 0, 2, 128, 4, 128, 6, 128,
0, 2, 4, 128, 6, 128, 8, 128,
128, 128, 128, 0, 2, 128, 4, 128,
0, 128, 128, 2, 4, 128, 6, 128,
128, 0, 128, 2, 4, 128, 6, 128,
0, 2, 128, 4, 6, 128, 8, 128,
128, 128, 0, 2, 4, 128, 6, 128,
0, 128, 2, 4, 6, 128, 8, 128,
128, 0, 2, 4, 6, 128, 8, 128,
0, 2, 4, 6, 8, 128, 10, 128,
128, 128, 128, 128, 128, 0, 2, 128,
0, 128, 128, 128, 128, 2, 4, 128,
128, 0, 128, 128, 128, 2, 4, 128,
0, 2, 128, 128, 128, 4, 6, 128,
128, 128, 0, 128, 128, 2, 4, 128,
0, 128, 2, 128, 128, 4, 6, 128,
128, 0, 2, 128, 128, 4, 6, 128,
0, 2, 4, 128, 128, 6, 8, 128,
128, 128, 128, 0, 128, 2, 4, 128,
0, 128, 128, 2, 128, 4, 6, 128,
128, 0, 128, 2, 128, 4, 6, 128,
0, 2, 128, 4, 128, 6, 8, 128,
128, 128, 0, 2, 128, 4, 6, 128,
0, 128, 2, 4, 128, 6, 8, 128,
128, 0, 2, 4, 128, 6, 8, 128,
0, 2, 4, 6, 128, 8, 10, 128,
128, 128, 128, 128, 0, 2, 4, 128,
0, 128, 128, 128, 2, 4, 6, 128,
128, 0, 128, 128, 2, 4, 6, 128,
0, 2, 128, 128, 4, 6, 8, 128,
128, 128, 0, 128, 2, 4, 6, 128,
0, 128, 2, 128, 4, 6, 8, 128,
128, 0, 2, 128, 4, 6, 8, 128,
0, 2, 4, 128, 6, 8, 10, 128,
128, 128, 128, 0, 2, 4, 6, 128,
0, 128, 128, 2, 4, 6, 8, 128,
128, 0, 128, 2, 4, 6, 8, 128,
0, 2, 128, 4, 6, 8, 10, 128,
128, 128, 0, 2, 4, 6, 8, 128,
0, 128, 2, 4, 6, 8, 10, 128,
128, 0, 2, 4, 6, 8, 10, 128,
0, 2, 4, 6, 8, 10, 12, 128,
128, 128, 128, 128, 128, 128, 128, 0,
0, 128, 128, 128, 128, 128, 128, 2,
128, 0, 128, 128, 128, 128, 128, 2,
0, 2, 128, 128, 128, 128, 128, 4,
128, 128, 0, 128, 128, 128, 128, 2,
0, 128, 2, 128, 128, 128, 128, 4,
128, 0, 2, 128, 128, 128, 128, 4,
0, 2, 4, 128, 128, 128, 128, 6,
128, 128, 128, 0, 128, 128, 128, 2,
0, 128, 128, 2, 128, 128, 128, 4,
128, 0, 128, 2, 128, 128, 128, 4,
0, 2, 128, 4, 128, 128, 128, 6,
128, 128, 0, 2, 128, 128, 128, 4,
0, 128, 2, 4, 128, 128, 128, 6,
128, 0, 2, 4, 128, 128, 128, 6,
0, 2, 4, 6, 128, 128, 128, 8,
128, 128, 128, 128, 0, 128, 128, 2,
0, 128, 128, 128, 2, 128, 128, 4,
128, 0, 128, 128, 2, 128, 128, 4,
0, 2, 128, 128, 4, 128, 128, 6,
128, 128, 0, 128, 2, 128, 128, 4,
0, 128, 2, 128, 4, 128, 128, 6,
128, 0, 2, 128, 4, 128, 128, 6,
0, 2, 4, 128, 6, 128, 128, 8,
128, 128, 128, 0, 2, 128, 128, 4,
0, 128, 128, 2, 4, 128, 128, 6,
128, 0, 128, 2, 4, 128, 128, 6,
0, 2, 128, 4, 6, 128, 128, 8,
128, 128, 0, 2, 4, 128, 128, 6,
0, 128, 2, 4, 6, 128, 128, 8,
128, 0, 2, 4, 6, 128, 128, 8,
0, 2, 4, 6, 8, 128, 128, 10,
128, 128, 128, 128, 128, 0, 128, 2,
0, 128, 128, 128, 128, 2, 128, 4,
128, 0, 128, 128, 128, 2, 128, 4,
0, 2, 128, 128, 128, 4, 128, 6,
128, 128, 0, 128, 128, 2, 128, 4,
0, 128, 2, 128, 128, 4, 128, 6,
128, 0, 2, 128, 128, 4, 128, 6,
0, 2, 4, 128, 128, 6, 128, 8,
128, 128, 128, 0, 128, 2, 128, 4,
0, 128, 128, 2, 128, 4, 128, 6,
128, 0, 128, 2, 128, 4, 128, 6,
0, 2, 128, 4, 128, 6, 128, 8,
128, 128, 0, 2, 128, 4, 128, 6,
0, 128, 2, 4, 128, 6, 128, 8,
128, 0, 2, 4, 128, 6, 128, 8,
0, 2, 4, 6, 128, 8, 128, 10,
128, 128, 128, 128, 0, 2, 128, 4,
0, 128, 128, 128, 2, 4, 128, 6,
128, 0, 128, 128, 2, 4, 128, 6,
0, 2, 128, 128, 4, 6, 128, 8,
128, 128, 0, 128, 2, 4, 128, 6,
0, 128, 2, 128, 4, 6, 128, 8,
128, 0, 2, 128, 4, 6, 128, 8,
0, 2, 4, 128, 6, 8, 128, 10,
128, 128, 128, 0, 2, 4, 128, 6,
0, 128, 128, 2, 4, 6, 128, 8,
128, 0, 128, 2, 4, 6, 128, 8,
0, 2, 128, 4, 6, 8, 128, 10,
128, 128, 0, 2, 4, 6, 128, 8,
0, 128, 2, 4, 6, 8, 128, 10,
128, 0, 2, 4, 6, 8, 128, 10,
0, 2, 4, 6, 8, 10, 128, 12,
128, 128, 128, 128, 128, 128, 0, 2,
0, 128, 128, 128, 128, 128, 2, 4,
128, 0, 128, 128, 128, 128, 2, 4,
0, 2, 128, 128, 128, 128, 4, 6,
128, 128, 0, 128, 128, 128, 2, 4,
0, 128, 2, 128, 128, 128, 4, 6,
128, 0, 2, 128, 128, 128, 4, 6,
0, 2, 4, 128, 128, 128, 6, 8,
128, 128, 128, 0, 128, 128, 2, 4,
0, 128, 128, 2, 128, 128, 4, 6,
128, 0, 128, 2, 128, 128, 4, 6,
0, 2, 128, 4, 128, 128, 6, 8,
128, 128, 0, 2, 128, 128, 4, 6,
0, 128, 2, 4, 128, 128, 6, 8,
128, 0, 2, 4, 128, 128, 6, 8,
0, 2, 4, 6, 128, 128, 8, 10,
128, 128, 128, 128, 0, 128, 2, 4,
0, 128, 128, 128, 2, 128, 4, 6,
128, 0, 128, 128, 2, 128, 4, 6,
0, 2, 128, 128, 4, 128, 6, 8,
128, 128, 0, 128, 2, 128, 4, 6,
0, 128, 2, 128, 4, 128, 6, 8,
128, 0, 2, 128, 4, 128, 6, 8,
0, 2, 4, 128, 6, 128, 8, 10,
128, 128, 128, 0, 2, 128, 4, 6,
0, 128, 128, 2, 4, 128, 6, 8,
128, 0, 128, 2, 4, 128, 6, 8,
0, 2, 128, 4, 6, 128, 8, 10,
128, 128, 0, 2, 4, 128, 6, 8,
0, 128, 2, 4, 6, 128, 8, 10,
128, 0, 2, 4, 6, 128, 8, 10,
0, 2, 4, 6, 8, 128, 10, 12,
128, 128, 128, 128, 128, 0, 2, 4,
0, 128, 128, 128, 128, 2, 4, 6,
128, 0, 128, 128, 128, 2, 4, 6,
0, 2, 128, 128, 128, 4, 6, 8,
128, 128, 0, 128, 128, 2, 4, 6,
0, 128, 2, 128, 128, 4, 6, 8,
128, 0, 2, 128, 128, 4, 6, 8,
0, 2, 4, 128, 128, 6, 8, 10,
128, 128, 128, 0, 128, 2, 4, 6,
0, 128, 128, 2, 128, 4, 6, 8,
128, 0, 128, 2, 128, 4, 6, 8,
0, 2, 128, 4, 128, 6, 8, 10,
128, 128, 0, 2, 128, 4, 6, 8,
0, 128, 2, 4, 128, 6, 8, 10,
128, 0, 2, 4, 128, 6, 8, 10,
0, 2, 4, 6, 128, 8, 10, 12,
128, 128, 128, 128, 0, 2, 4, 6,
0, 128, 128, 128, 2, 4, 6, 8,
128, 0, 128, 128, 2, 4, 6, 8,
0, 2, 128, 128, 4, 6, 8, 10,
128, 128, 0, 128, 2, 4, 6, 8,
0, 128, 2, 128, 4, 6, 8, 10,
128, 0, 2, 128, 4, 6, 8, 10,
0, 2, 4, 128, 6, 8, 10, 12,
128, 128, 128, 0, 2, 4, 6, 8,
0, 128, 128, 2, 4, 6, 8, 10,
128, 0, 128, 2, 4, 6, 8, 10,
0, 2, 128, 4, 6, 8, 10, 12,
128, 128, 0, 2, 4, 6, 8, 10,
0, 128, 2, 4, 6, 8, 10, 12,
128, 0, 2, 4, 6, 8, 10, 12,
0, 2, 4, 6, 8, 10, 12, 14};
const Twice<decltype(du8)> du8x2;
const Vec128<uint8_t, 2 * N> indices8 =
ZeroExtendVector(du8x2, Load(du8, table + mask_bits * 8));
const Vec128<uint16_t, N> indices16 =
BitCast(du, InterleaveLower(du8x2, indices8, indices8));
const Vec128<uint16_t, N> byte_indices = Add(indices16, Set(du, 0x0100));
return BitCast(d, TableLookupBytesOr0(v, byte_indices));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Expand(Vec128<T, N> v, Mask128<T, N> mask) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const uint64_t mask_bits = detail::BitsFromMask(mask);
alignas(16) static constexpr uint32_t packed_array[16] = {
0x0000ffff, 0x0000fff0, 0x0000ff0f, 0x0000ff10, 0x0000f0ff, 0x0000f1f0,
0x0000f10f, 0x0000f210, 0x00000fff, 0x00001ff0, 0x00001f0f, 0x00002f10,
0x000010ff, 0x000021f0, 0x0000210f, 0x00003210};
const Vec128<uint32_t, N> packed = Set(du, packed_array[mask_bits]);
alignas(16) static constexpr uint32_t shifts[4] = {0, 4, 8, 12};
Vec128<uint32_t, N> indices = packed >> Load(du, shifts);
indices = And(indices, Set(du, N - 1));
const Vec128<uint32_t, N> expand =
TableLookupLanes(BitCast(du, v), IndicesFromVec(du, indices));
return IfThenElseZero(mask, BitCast(d, expand));
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Expand(Vec128<T> v, Mask128<T> mask) {
return IfThenElseZero(mask, Compress(v, mask));
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> Expand(Vec128<T, 1> v, Mask128<T, 1> mask) {
return IfThenElseZero(mask, v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> LoadExpand(MFromD<D> mask, D d,
const TFromD<D>* __restrict__ unaligned) {
return Expand(LoadU(d, unaligned), mask);
}
template <class D>
using IndicesFromD = decltype(IndicesFromVec(D(), Zero(RebindToUnsigned<D>())));
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TwoTablesLookupLanes(D , VFromD<D> a, VFromD<D> b,
IndicesFromD<D> idx) {
return TwoTablesLookupLanes(a, b, idx);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse2(D d, VFromD<D> v) {
const Repartition<uint16_t, decltype(d)> du16;
return BitCast(d, RotateRight<8>(BitCast(du16, v)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse4(D d, VFromD<D> v) {
const Repartition<uint16_t, decltype(d)> du16;
return BitCast(d, Reverse2(du16, BitCast(du16, Reverse2(d, v))));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse8(D d, VFromD<D> v) {
const Repartition<uint32_t, D> du32;
return BitCast(d, Reverse2(du32, BitCast(du32, Reverse4(d, v))));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseLaneBytes(V v) {
const DFromV<V> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, Reverse2(du8, BitCast(du8, v)));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseLaneBytes(V v) {
const DFromV<V> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, Reverse4(du8, BitCast(du8, v)));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseLaneBytes(V v) {
const DFromV<V> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, Reverse8(du8, BitCast(du8, v)));
}
namespace detail {
template <int kShiftAmt, int kShrResultMask, class V,
hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromD<DFromV<V> >) > 2 - 1)>* = nullptr>
inline __attribute__((always_inline)) V UI8ReverseBitsStep(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const Repartition<uint16_t, decltype(d)> d_shift;
const auto v_to_shift = BitCast(d_shift, v);
const auto shl_result = BitCast(d, ShiftLeft<kShiftAmt>(v_to_shift));
const auto shr_result = BitCast(d, ShiftRight<kShiftAmt>(v_to_shift));
const auto shr_result_mask =
BitCast(d, Set(du, static_cast<uint8_t>(kShrResultMask)));
return Or(And(shr_result, shr_result_mask),
AndNot(shr_result_mask, shl_result));
}
template <int kShiftAmt, int kShrResultMask, class V,
hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromD<DFromV<V> >) == 1>* = nullptr>
inline __attribute__((always_inline)) V UI8ReverseBitsStep(V v) {
return V{UI8ReverseBitsStep<kShiftAmt, kShrResultMask>(Vec128<uint8_t>{v.raw})
.raw};
}
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseBits(V v) {
auto result = detail::UI8ReverseBitsStep<1, 0x55>(v);
result = detail::UI8ReverseBitsStep<2, 0x33>(result);
result = detail::UI8ReverseBitsStep<4, 0x0F>(result);
return result;
}
template <class V, hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 2) | (1 << 4) | (1 << 8))) != 0>* = nullptr,
hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseBits(V v) {
const DFromV<decltype(v)> d;
const Repartition<uint8_t, decltype(d)> du8;
return ReverseLaneBytes(BitCast(d, ReverseBits(BitCast(du8, v))));
}
namespace detail {
template <class V>
inline __attribute__((always_inline)) V Per2LaneBlockShuffle(hwy::SizeTag<0> , V v) {
return DupEven(v);
}
template <class V>
inline __attribute__((always_inline)) V Per2LaneBlockShuffle(hwy::SizeTag<1> , V v) {
const DFromV<decltype(v)> d;
return Reverse2(d, v);
}
template <class V>
inline __attribute__((always_inline)) V Per2LaneBlockShuffle(hwy::SizeTag<2> , V v) {
return v;
}
template <class V>
inline __attribute__((always_inline)) V Per2LaneBlockShuffle(hwy::SizeTag<3> , V v) {
return DupOdd(v);
}
inline __attribute__((always_inline)) uint32_t U8x4Per4LaneBlkIndices(const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
return static_cast<uint32_t>((idx3 << 24) | (idx2 << 16) | (idx1 << 8) |
idx0);
}
template <class D>
inline __attribute__((always_inline)) Vec<D> TblLookupPer4LaneBlkU8IdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const Repartition<uint32_t, D> du32;
return ResizeBitCast(
d, Set(du32, U8x4Per4LaneBlkIndices(idx3, idx2, idx1, idx0)));
}
template <class V, hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2) | (1 << 4))) != 0>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlkShufDoTblLookup(V v, V idx) {
const DFromV<decltype(v)> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, TableLookupBytes(BitCast(du8, v), BitCast(du8, idx)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
inline __attribute__((always_inline)) Vec<D> TblLookupPer4LaneBlkShufIdx(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const Repartition<uint32_t, decltype(d)> du32;
const uint32_t idx3210 = U8x4Per4LaneBlkIndices(idx3, idx2, idx1, idx0);
const auto v_byte_idx = Per4LaneBlkShufDupSet4xU32(
du32, static_cast<uint32_t>(idx3210 + 0x0C0C0C0C),
static_cast<uint32_t>(idx3210 + 0x08080808),
static_cast<uint32_t>(idx3210 + 0x04040404),
static_cast<uint32_t>(idx3210));
return ResizeBitCast(d, v_byte_idx);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
inline __attribute__((always_inline)) Vec<D> TblLookupPer4LaneBlkShufIdx(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const Repartition<uint32_t, decltype(d)> du32;
const uint32_t idx10 = static_cast<uint32_t>((idx1 << 16) | idx0);
const uint32_t idx32 = static_cast<uint32_t>((idx3 << 16) | idx2);
constexpr uint32_t kLaneByteOffsets{0x01000100};
constexpr uint32_t kHiLaneByteOffsets{kLaneByteOffsets + 0x08080808u};
const auto v_byte_idx = Per4LaneBlkShufDupSet4xU32(
du32, static_cast<uint32_t>(idx32 * 0x0202u + kHiLaneByteOffsets),
static_cast<uint32_t>(idx10 * 0x0202u + kHiLaneByteOffsets),
static_cast<uint32_t>(idx32 * 0x0202u + kLaneByteOffsets),
static_cast<uint32_t>(idx10 * 0x0202u + kLaneByteOffsets));
return ResizeBitCast(d, v_byte_idx);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec<D> TblLookupPer4LaneBlkShufIdx(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const Repartition<uint32_t, decltype(d)> du32;
constexpr uint32_t kLaneByteOffsets{0x03020100};
const auto v_byte_idx = Per4LaneBlkShufDupSet4xU32(
du32, static_cast<uint32_t>(idx3 * 0x04040404u + kLaneByteOffsets),
static_cast<uint32_t>(idx2 * 0x04040404u + kLaneByteOffsets),
static_cast<uint32_t>(idx1 * 0x04040404u + kLaneByteOffsets),
static_cast<uint32_t>(idx0 * 0x04040404u + kLaneByteOffsets));
return ResizeBitCast(d, v_byte_idx);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> TblLookupPer4LaneBlkIdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
return TblLookupPer4LaneBlkU8IdxInBlk(d, idx3, idx2, idx1, idx0);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> TblLookupPer4LaneBlkIdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const uint16_t u16_idx0 = static_cast<uint16_t>(idx0);
const uint16_t u16_idx1 = static_cast<uint16_t>(idx1);
const uint16_t u16_idx2 = static_cast<uint16_t>(idx2);
const uint16_t u16_idx3 = static_cast<uint16_t>(idx3);
alignas(16)
const uint16_t indices[8] = {u16_idx0, u16_idx1, u16_idx2, u16_idx3,
u16_idx0, u16_idx1, u16_idx2, u16_idx3};
constexpr size_t kMinLanesToLoad = 8;
constexpr size_t kNumToLoad = ((D::kPrivateLanes) > (kMinLanesToLoad) ? (D::kPrivateLanes) : (kMinLanesToLoad));
const CappedTag<uint16_t, kNumToLoad> d_load;
return ResizeBitCast(d, LoadDup128(d_load, indices));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> TblLookupPer4LaneBlkIdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
return Per4LaneBlkShufDupSet4xU32(d, idx3, idx2, idx1, idx0);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> TblLookupPer4LaneBlkIdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const RebindToUnsigned<decltype(d)> du;
const Rebind<uint32_t, decltype(d)> du32;
return BitCast(d, PromoteTo(du, Per4LaneBlkShufDupSet4xU32(du32, idx3, idx2,
idx1, idx0)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
inline __attribute__((always_inline)) IndicesFromD<D> TblLookupPer4LaneBlkShufIdx(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
auto idx_in_blk = TblLookupPer4LaneBlkIdxInBlk(du, idx3, idx2, idx1, idx0);
constexpr size_t kN = D::kPrivateLanes;
if (kN < 4) {
idx_in_blk = And(idx_in_blk, Set(du, static_cast<TU>(kN - 1)));
}
const auto blk_offsets =
And(Iota(du, TU{0}), Set(du, static_cast<TU>(~TU{3})));
return IndicesFromVec(d, Add(idx_in_blk, blk_offsets));
}
template <class V, hwy::EnableIf<sizeof(TFromD<DFromV<V> >) == (8)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlkShufDoTblLookup(V v, IndicesFromD<DFromV<V>> idx) {
return TableLookupLanes(v, idx);
}
template <class V>
inline __attribute__((always_inline)) V TblLookupPer4LaneBlkShuf(V v, size_t idx3210) {
const DFromV<decltype(v)> d;
const uint32_t idx3 = static_cast<uint32_t>((idx3210 >> 6) & 3);
const uint32_t idx2 = static_cast<uint32_t>((idx3210 >> 4) & 3);
const uint32_t idx1 = static_cast<uint32_t>((idx3210 >> 2) & 3);
const uint32_t idx0 = static_cast<uint32_t>(idx3210 & 3);
const auto idx = TblLookupPer4LaneBlkShufIdx(d, idx3, idx2, idx1, idx0);
return Per4LaneBlkShufDoTblLookup(v, idx);
}
template <size_t kIdx3210, size_t kLaneSize, size_t kVectSize, class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> ,
hwy::SizeTag<kLaneSize> ,
hwy::SizeTag<kVectSize> ,
V v) {
return TblLookupPer4LaneBlkShuf(v, kIdx3210);
}
template <class V>
inline __attribute__((always_inline)) VFromD<RepartitionToWide<DFromV<V>>> Per4LaneBlockShufCastToWide(
hwy::FloatTag , hwy::SizeTag<4> , V v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> dw;
return BitCast(dw, v);
}
template <size_t kLaneSize, class V>
inline __attribute__((always_inline)) VFromD<RepartitionToWide<RebindToUnsigned<DFromV<V>>>>
Per4LaneBlockShufCastToWide(hwy::FloatTag ,
hwy::SizeTag<kLaneSize> , V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const RepartitionToWide<decltype(du)> dw;
return BitCast(dw, v);
}
template <size_t kLaneSize, class V>
inline __attribute__((always_inline)) VFromD<RepartitionToWide<DFromV<V>>> Per4LaneBlockShufCastToWide(
hwy::NonFloatTag ,
hwy::SizeTag<kLaneSize> , V v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> dw;
return BitCast(dw, v);
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x1B> , V v) {
const DFromV<decltype(v)> d;
return Reverse4(d, v);
}
template <class V,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2) | (1 ? (1 << 4) : 0))) != 0>* = nullptr
>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x44> , V v) {
const DFromV<decltype(v)> d;
const auto vw = Per4LaneBlockShufCastToWide(
hwy::IsFloatTag<TFromV<V>>(), hwy::SizeTag<sizeof(TFromV<V>)>(), v);
return BitCast(d, DupEven(vw));
}
template <class V,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2) | (1 ? (1 << 4) : 0))) != 0>* = nullptr
>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x4E> , V v) {
const DFromV<decltype(v)> d;
const auto vw = Per4LaneBlockShufCastToWide(
hwy::IsFloatTag<TFromV<V>>(), hwy::SizeTag<sizeof(TFromV<V>)>(), v);
const DFromV<decltype(vw)> dw;
return BitCast(d, Reverse2(dw, vw));
}
template <class V, hwy::EnableIf<(DFromV<V>::kPrivateLanes == 4)>* = nullptr,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2))) != 0>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x50> , V v) {
const DFromV<decltype(v)> d;
return InterleaveLower(d, v, v);
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (4)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x50> , V v) {
const DFromV<decltype(v)> d;
return InterleaveLower(d, v, v);
}
template <class V, hwy::EnableIf<(DFromV<V>::kPrivateLanes == 4)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x88> , V v) {
const DFromV<decltype(v)> d;
return ConcatEven(d, v, v);
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xA0> , V v) {
return DupEven(v);
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xB1> , V v) {
const DFromV<decltype(v)> d;
return Reverse2(d, v);
}
template <class V, hwy::EnableIf<(DFromV<V>::kPrivateLanes == 4)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xDD> , V v) {
const DFromV<decltype(v)> d;
return ConcatOdd(d, v, v);
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xE4> , V v) {
return v;
}
template <class V,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2) | (1 ? (1 << 4) : 0))) != 0>* = nullptr
>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xEE> , V v) {
const DFromV<decltype(v)> d;
const auto vw = Per4LaneBlockShufCastToWide(
hwy::IsFloatTag<TFromV<V>>(), hwy::SizeTag<sizeof(TFromV<V>)>(), v);
return BitCast(d, DupOdd(vw));
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xF5> , V v) {
return DupOdd(v);
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (4)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xFA> , V v) {
const DFromV<decltype(v)> d;
return InterleaveUpper(d, v, v);
}
template <size_t kIdx3210, class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> idx_3210_tag, V v) {
const DFromV<decltype(v)> d;
return Per4LaneBlockShuffle(idx_3210_tag, hwy::SizeTag<sizeof(TFromV<V>)>(),
hwy::SizeTag<d.MaxBytes()>(), v);
}
}
template <size_t kIdx3, size_t kIdx2, size_t kIdx1, size_t kIdx0, class V,
hwy::EnableIf<(DFromV<V>::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Per4LaneBlockShuffle(V v) {
static_assert(kIdx0 <= 3, "kIdx0 <= 3 must be true");
static_assert(kIdx1 <= 3, "kIdx1 <= 3 must be true");
static_assert(kIdx2 <= 3, "kIdx2 <= 3 must be true");
static_assert(kIdx3 <= 3, "kIdx3 <= 3 must be true");
return v;
}
template <size_t kIdx3, size_t kIdx2, size_t kIdx1, size_t kIdx0, class V,
hwy::EnableIf<(DFromV<V>::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Per4LaneBlockShuffle(V v) {
static_assert(kIdx0 <= 3, "kIdx0 <= 3 must be true");
static_assert(kIdx1 <= 3, "kIdx1 <= 3 must be true");
static_assert(kIdx2 <= 3, "kIdx2 <= 3 must be true");
static_assert(kIdx3 <= 3, "kIdx3 <= 3 must be true");
constexpr bool isReverse2 = (kIdx0 == 1 || kIdx1 == 0) && (kIdx0 != kIdx1);
constexpr size_t kPer2BlkIdx0 = (kIdx0 <= 1) ? kIdx0 : (isReverse2 ? 1 : 0);
constexpr size_t kPer2BlkIdx1 = (kIdx1 <= 1) ? kIdx1 : (isReverse2 ? 0 : 1);
constexpr size_t kIdx10 = (kPer2BlkIdx1 << 1) | kPer2BlkIdx0;
static_assert(kIdx10 <= 3, "kIdx10 <= 3 must be true");
return detail::Per2LaneBlockShuffle(hwy::SizeTag<kIdx10>(), v);
}
template <size_t kIdx3, size_t kIdx2, size_t kIdx1, size_t kIdx0, class V,
hwy::EnableIf<(DFromV<V>::kPrivateLanes > 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Per4LaneBlockShuffle(V v) {
static_assert(kIdx0 <= 3, "kIdx0 <= 3 must be true");
static_assert(kIdx1 <= 3, "kIdx1 <= 3 must be true");
static_assert(kIdx2 <= 3, "kIdx2 <= 3 must be true");
static_assert(kIdx3 <= 3, "kIdx3 <= 3 must be true");
constexpr size_t kIdx3210 =
(kIdx3 << 6) | (kIdx2 << 4) | (kIdx1 << 2) | kIdx0;
return detail::Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210>(), v);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t Blocks(D d) {
return (d.MaxBytes() <= 16) ? 1 : ((Lanes(d) * sizeof(TFromD<D>) + 15) / 16);
}
template <int kBlockIdx, class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V InsertBlock(V , V blk_to_insert) {
static_assert(kBlockIdx == 0, "Invalid block index");
return blk_to_insert;
}
template <int kBlockIdx, class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ExtractBlock(V v) {
static_assert(kBlockIdx == 0, "Invalid block index");
return v;
}
template <int kBlockIdx, class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V BroadcastBlock(V v) {
static_assert(kBlockIdx == 0, "Invalid block index");
return v;
}
template<int kLane, class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V BroadcastLane(V v) {
return Broadcast<kLane>(v);
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Slide1Up(D d, VFromD<D> ) {
return Zero(d);
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Slide1Down(D d, VFromD<D> ) {
return Zero(d);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Slide1Up(D d, VFromD<D> v) {
return ShiftLeftLanes<1>(d, v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Slide1Down(D d, VFromD<D> v) {
return ShiftRightLanes<1>(d, v);
}
template <int kBlocks, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpBlocks(D , VFromD<D> v) {
static_assert(kBlocks == 0, "kBlocks == 0 must be true");
return v;
}
template <int kBlocks, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownBlocks(D , VFromD<D> v) {
static_assert(kBlocks == 0, "kBlocks == 0 must be true");
return v;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Add(V a, V b) {
return a + b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Sub(V a, V b) {
return a - b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Mul(V a, V b) {
return a * b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Div(V a, V b) {
return a / b;
}
template <class V>
V Shl(V a, V b) {
return a << b;
}
template <class V>
V Shr(V a, V b) {
return a >> b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Eq(V a, V b) -> decltype(a == b) {
return a == b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Ne(V a, V b) -> decltype(a == b) {
return a != b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Lt(V a, V b) -> decltype(a == b) {
return a < b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Gt(V a, V b) -> decltype(a == b) {
return a > b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Ge(V a, V b) -> decltype(a == b) {
return a >= b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Le(V a, V b) -> decltype(a == b) {
return a <= b;
}
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC push_options
#pragma GCC target "sse2"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSE2 {
template <class D, class V>
inline __attribute__((always_inline)) V Acos(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAcos(const D d, VecArg<V> x) {
return Acos(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Acosh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAcosh(const D d, VecArg<V> x) {
return Acosh(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Asin(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAsin(const D d, VecArg<V> x) {
return Asin(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Asinh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAsinh(const D d, VecArg<V> x) {
return Asinh(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Atan(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAtan(const D d, VecArg<V> x) {
return Atan(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Atanh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAtanh(const D d, VecArg<V> x) {
return Atanh(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Cos(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallCos(const D d, VecArg<V> x) {
return Cos(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Exp(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallExp(const D d, VecArg<V> x) {
return Exp(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Expm1(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallExpm1(const D d, VecArg<V> x) {
return Expm1(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallLog(const D d, VecArg<V> x) {
return Log(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log10(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallLog10(const D d, VecArg<V> x) {
return Log10(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log1p(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallLog1p(const D d, VecArg<V> x) {
return Log1p(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log2(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallLog2(const D d, VecArg<V> x) {
return Log2(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Sin(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallSin(const D d, VecArg<V> x) {
return Sin(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Sinh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallSinh(const D d, VecArg<V> x) {
return Sinh(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Tanh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallTanh(const D d, VecArg<V> x) {
return Tanh(d, x);
}
namespace impl {
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1) {
return MulAdd(c1, x, c0);
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2) {
T x2 = Mul(x, x);
return MulAdd(x2, c2, MulAdd(c1, x, c0));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3) {
T x2 = Mul(x, x);
return MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
return MulAdd(x4, c4, MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
return MulAdd(x4, MulAdd(c5, x, c4),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
return MulAdd(x4, MulAdd(x2, c6, MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
return MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8, c8,
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8, MulAdd(c9, x, c8),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8, MulAdd(x2, c10, MulAdd(c9, x, c8)),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8, MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8)),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(
x8, MulAdd(x4, c12, MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8,
MulAdd(x4, MulAdd(c13, x, c12),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8,
MulAdd(x4, MulAdd(x2, c14, MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14, T c15) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8,
MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14, T c15, T c16) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
T x16 = Mul(x8, x8);
return MulAdd(
x16, c16,
MulAdd(x8,
MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14, T c15, T c16, T c17) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
T x16 = Mul(x8, x8);
return MulAdd(
x16, MulAdd(c17, x, c16),
MulAdd(x8,
MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14, T c15, T c16, T c17,
T c18) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
T x16 = Mul(x8, x8);
return MulAdd(
x16, MulAdd(x2, c18, MulAdd(c17, x, c16)),
MulAdd(x8,
MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))));
}
template <class FloatOrDouble>
struct AsinImpl {};
template <class FloatOrDouble>
struct AtanImpl {};
template <class FloatOrDouble>
struct CosSinImpl {};
template <class FloatOrDouble>
struct ExpImpl {};
template <class FloatOrDouble>
struct LogImpl {};
template <>
struct AsinImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) V AsinPoly(D d, V x2, V ) {
const auto k0 = Set(d, +0.1666677296f);
const auto k1 = Set(d, +0.07495029271f);
const auto k2 = Set(d, +0.04547423869f);
const auto k3 = Set(d, +0.02424046025f);
const auto k4 = Set(d, +0.04197454825f);
return Estrin(x2, k0, k1, k2, k3, k4);
}
};
template <>
struct AsinImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) V AsinPoly(D d, V x2, V ) {
const auto k0 = Set(d, +0.1666666666666497543);
const auto k1 = Set(d, +0.07500000000378581611);
const auto k2 = Set(d, +0.04464285681377102438);
const auto k3 = Set(d, +0.03038195928038132237);
const auto k4 = Set(d, +0.02237176181932048341);
const auto k5 = Set(d, +0.01735956991223614604);
const auto k6 = Set(d, +0.01388715184501609218);
const auto k7 = Set(d, +0.01215360525577377331);
const auto k8 = Set(d, +0.006606077476277170610);
const auto k9 = Set(d, +0.01929045477267910674);
const auto k10 = Set(d, -0.01581918243329996643);
const auto k11 = Set(d, +0.03161587650653934628);
return Estrin(x2, k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11);
}
};
template <>
struct AtanImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) V AtanPoly(D d, V x) {
const auto k0 = Set(d, -0.333331018686294555664062f);
const auto k1 = Set(d, +0.199926957488059997558594f);
const auto k2 = Set(d, -0.142027363181114196777344f);
const auto k3 = Set(d, +0.106347933411598205566406f);
const auto k4 = Set(d, -0.0748900920152664184570312f);
const auto k5 = Set(d, +0.0425049886107444763183594f);
const auto k6 = Set(d, -0.0159569028764963150024414f);
const auto k7 = Set(d, +0.00282363896258175373077393f);
const auto y = Mul(x, x);
return MulAdd(Estrin(y, k0, k1, k2, k3, k4, k5, k6, k7), Mul(y, x), x);
}
};
template <>
struct AtanImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) V AtanPoly(D d, V x) {
const auto k0 = Set(d, -0.333333333333311110369124);
const auto k1 = Set(d, +0.199999999996591265594148);
const auto k2 = Set(d, -0.14285714266771329383765);
const auto k3 = Set(d, +0.111111105648261418443745);
const auto k4 = Set(d, -0.090908995008245008229153);
const auto k5 = Set(d, +0.0769219538311769618355029);
const auto k6 = Set(d, -0.0666573579361080525984562);
const auto k7 = Set(d, +0.0587666392926673580854313);
const auto k8 = Set(d, -0.0523674852303482457616113);
const auto k9 = Set(d, +0.0466667150077840625632675);
const auto k10 = Set(d, -0.0407629191276836500001934);
const auto k11 = Set(d, +0.0337852580001353069993897);
const auto k12 = Set(d, -0.0254517624932312641616861);
const auto k13 = Set(d, +0.016599329773529201970117);
const auto k14 = Set(d, -0.00889896195887655491740809);
const auto k15 = Set(d, +0.00370026744188713119232403);
const auto k16 = Set(d, -0.00110611831486672482563471);
const auto k17 = Set(d, +0.000209850076645816976906797);
const auto k18 = Set(d, -1.88796008463073496563746e-5);
const auto y = Mul(x, x);
return MulAdd(Estrin(y, k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11,
k12, k13, k14, k15, k16, k17, k18),
Mul(y, x), x);
}
};
template <>
struct CosSinImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> ToInt32(D , V x) {
return ConvertTo(Rebind<int32_t, D>(), x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Poly(D d, V x) {
const auto k0 = Set(d, -1.66666597127914428710938e-1f);
const auto k1 = Set(d, +8.33307858556509017944336e-3f);
const auto k2 = Set(d, -1.981069071916863322258e-4f);
const auto k3 = Set(d, +2.6083159809786593541503e-6f);
const auto y = Mul(x, x);
return MulAdd(Estrin(y, k0, k1, k2, k3), Mul(y, x), x);
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V CosReduce(D d, V x, VI32 q) {
const V kHalfPiPart0f = Set(d, -0.5f * 3.140625f);
const V kHalfPiPart1f = Set(d, -0.5f * 0.0009670257568359375f);
const V kHalfPiPart2f = Set(d, -0.5f * 6.2771141529083251953e-7f);
const V kHalfPiPart3f = Set(d, -0.5f * 1.2154201256553420762e-10f);
const V qf = ConvertTo(d, q);
x = MulAdd(qf, kHalfPiPart0f, x);
x = MulAdd(qf, kHalfPiPart1f, x);
x = MulAdd(qf, kHalfPiPart2f, x);
x = MulAdd(qf, kHalfPiPart3f, x);
return x;
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V SinReduce(D d, V x, VI32 q) {
const V kPiPart0f = Set(d, -3.140625f);
const V kPiPart1f = Set(d, -0.0009670257568359375f);
const V kPiPart2f = Set(d, -6.2771141529083251953e-7f);
const V kPiPart3f = Set(d, -1.2154201256553420762e-10f);
const V qf = ConvertTo(d, q);
x = MulAdd(qf, kPiPart0f, x);
x = MulAdd(qf, kPiPart1f, x);
x = MulAdd(qf, kPiPart2f, x);
x = MulAdd(qf, kPiPart3f, x);
return x;
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<Rebind<float, D>> CosSignFromQuadrant(D d, VI32 q) {
const VI32 kTwo = Set(Rebind<int32_t, D>(), 2);
return BitCast(d, ShiftLeft<30>(AndNot(q, kTwo)));
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<Rebind<float, D>> SinSignFromQuadrant(D d, VI32 q) {
const VI32 kOne = Set(Rebind<int32_t, D>(), 1);
return BitCast(d, ShiftLeft<31>(And(q, kOne)));
}
};
template <>
struct CosSinImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> ToInt32(D , V x) {
return DemoteTo(Rebind<int32_t, D>(), x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Poly(D d, V x) {
const auto k0 = Set(d, -0.166666666666666657414808);
const auto k1 = Set(d, +0.00833333333333332974823815);
const auto k2 = Set(d, -0.000198412698412696162806809);
const auto k3 = Set(d, +2.75573192239198747630416e-6);
const auto k4 = Set(d, -2.50521083763502045810755e-8);
const auto k5 = Set(d, +1.60590430605664501629054e-10);
const auto k6 = Set(d, -7.64712219118158833288484e-13);
const auto k7 = Set(d, +2.81009972710863200091251e-15);
const auto k8 = Set(d, -7.97255955009037868891952e-18);
const auto y = Mul(x, x);
return MulAdd(Estrin(y, k0, k1, k2, k3, k4, k5, k6, k7, k8), Mul(y, x), x);
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V CosReduce(D d, V x, VI32 q) {
const V kHalfPiPart0d = Set(d, -0.5 * 3.1415926218032836914);
const V kHalfPiPart1d = Set(d, -0.5 * 3.1786509424591713469e-8);
const V kHalfPiPart2d = Set(d, -0.5 * 1.2246467864107188502e-16);
const V kHalfPiPart3d = Set(d, -0.5 * 1.2736634327021899816e-24);
const V qf = PromoteTo(d, q);
x = MulAdd(qf, kHalfPiPart0d, x);
x = MulAdd(qf, kHalfPiPart1d, x);
x = MulAdd(qf, kHalfPiPart2d, x);
x = MulAdd(qf, kHalfPiPart3d, x);
return x;
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V SinReduce(D d, V x, VI32 q) {
const V kPiPart0d = Set(d, -3.1415926218032836914);
const V kPiPart1d = Set(d, -3.1786509424591713469e-8);
const V kPiPart2d = Set(d, -1.2246467864107188502e-16);
const V kPiPart3d = Set(d, -1.2736634327021899816e-24);
const V qf = PromoteTo(d, q);
x = MulAdd(qf, kPiPart0d, x);
x = MulAdd(qf, kPiPart1d, x);
x = MulAdd(qf, kPiPart2d, x);
x = MulAdd(qf, kPiPart3d, x);
return x;
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<Rebind<double, D>> CosSignFromQuadrant(D d, VI32 q) {
const VI32 kTwo = Set(Rebind<int32_t, D>(), 2);
return BitCast(
d, ShiftLeft<62>(PromoteTo(Rebind<int64_t, D>(), AndNot(q, kTwo))));
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<Rebind<double, D>> SinSignFromQuadrant(D d, VI32 q) {
const VI32 kOne = Set(Rebind<int32_t, D>(), 1);
return BitCast(
d, ShiftLeft<63>(PromoteTo(Rebind<int64_t, D>(), And(q, kOne))));
}
};
template <>
struct ExpImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> ToInt32(D , V x) {
return ConvertTo(Rebind<int32_t, D>(), x);
}
template <class D, class V>
inline __attribute__((always_inline)) V ExpPoly(D d, V x) {
const auto k0 = Set(d, +0.5f);
const auto k1 = Set(d, +0.166666671633720397949219f);
const auto k2 = Set(d, +0.0416664853692054748535156f);
const auto k3 = Set(d, +0.00833336077630519866943359f);
const auto k4 = Set(d, +0.00139304355252534151077271f);
const auto k5 = Set(d, +0.000198527617612853646278381f);
return MulAdd(Estrin(x, k0, k1, k2, k3, k4, k5), Mul(x, x), x);
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<D> Pow2I(D d, VI32 x) {
const Rebind<int32_t, D> di32;
const VI32 kOffset = Set(di32, 0x7F);
return BitCast(d, ShiftLeft<23>(Add(x, kOffset)));
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V LoadExpShortRange(D d, V x, VI32 e) {
const VI32 y = ShiftRight<1>(e);
return Mul(Mul(x, Pow2I(d, y)), Pow2I(d, Sub(e, y)));
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V ExpReduce(D d, V x, VI32 q) {
const V kLn2Part0f = Set(d, -0.693145751953125f);
const V kLn2Part1f = Set(d, -1.428606765330187045e-6f);
const V qf = ConvertTo(d, q);
x = MulAdd(qf, kLn2Part0f, x);
x = MulAdd(qf, kLn2Part1f, x);
return x;
}
};
template <>
struct LogImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> Log2p1NoSubnormal(D , V x) {
const Rebind<int32_t, D> di32;
const Rebind<uint32_t, D> du32;
const auto kBias = Set(di32, 0x7F);
return Sub(BitCast(di32, ShiftRight<23>(BitCast(du32, x))), kBias);
}
template <class D, class V>
inline __attribute__((always_inline)) V LogPoly(D d, V x) {
const V k0 = Set(d, 0.66666662693f);
const V k1 = Set(d, 0.40000972152f);
const V k2 = Set(d, 0.28498786688f);
const V k3 = Set(d, 0.24279078841f);
const V x2 = Mul(x, x);
const V x4 = Mul(x2, x2);
return MulAdd(MulAdd(k2, x4, k0), x2, Mul(MulAdd(k3, x4, k1), x4));
}
};
template <>
struct ExpImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> ToInt32(D , V x) {
return DemoteTo(Rebind<int32_t, D>(), x);
}
template <class D, class V>
inline __attribute__((always_inline)) V ExpPoly(D d, V x) {
const auto k0 = Set(d, +0.5);
const auto k1 = Set(d, +0.166666666666666851703837);
const auto k2 = Set(d, +0.0416666666666665047591422);
const auto k3 = Set(d, +0.00833333333331652721664984);
const auto k4 = Set(d, +0.00138888888889774492207962);
const auto k5 = Set(d, +0.000198412698960509205564975);
const auto k6 = Set(d, +2.4801587159235472998791e-5);
const auto k7 = Set(d, +2.75572362911928827629423e-6);
const auto k8 = Set(d, +2.75573911234900471893338e-7);
const auto k9 = Set(d, +2.51112930892876518610661e-8);
const auto k10 = Set(d, +2.08860621107283687536341e-9);
return MulAdd(Estrin(x, k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10),
Mul(x, x), x);
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<D> Pow2I(D d, VI32 x) {
const Rebind<int32_t, D> di32;
const Rebind<int64_t, D> di64;
const VI32 kOffset = Set(di32, 0x3FF);
return BitCast(d, ShiftLeft<52>(PromoteTo(di64, Add(x, kOffset))));
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V LoadExpShortRange(D d, V x, VI32 e) {
const VI32 y = ShiftRight<1>(e);
return Mul(Mul(x, Pow2I(d, y)), Pow2I(d, Sub(e, y)));
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V ExpReduce(D d, V x, VI32 q) {
const V kLn2Part0d = Set(d, -0.6931471805596629565116018);
const V kLn2Part1d = Set(d, -0.28235290563031577122588448175e-12);
const V qf = PromoteTo(d, q);
x = MulAdd(qf, kLn2Part0d, x);
x = MulAdd(qf, kLn2Part1d, x);
return x;
}
};
template <>
struct LogImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int64_t, D>> Log2p1NoSubnormal(D , V x) {
const Rebind<int64_t, D> di64;
const Rebind<uint64_t, D> du64;
return Sub(BitCast(di64, ShiftRight<52>(BitCast(du64, x))),
Set(di64, 0x3FF));
}
template <class D, class V>
inline __attribute__((always_inline)) V LogPoly(D d, V x) {
const V k0 = Set(d, 0.6666666666666735130);
const V k1 = Set(d, 0.3999999999940941908);
const V k2 = Set(d, 0.2857142874366239149);
const V k3 = Set(d, 0.2222219843214978396);
const V k4 = Set(d, 0.1818357216161805012);
const V k5 = Set(d, 0.1531383769920937332);
const V k6 = Set(d, 0.1479819860511658591);
const V x2 = Mul(x, x);
const V x4 = Mul(x2, x2);
return MulAdd(MulAdd(MulAdd(MulAdd(k6, x4, k4), x4, k2), x4, k0), x2,
(Mul(MulAdd(MulAdd(k5, x4, k3), x4, k1), x4)));
}
};
template <class D, class V, bool kAllowSubnormals = true>
inline __attribute__((always_inline)) V Log(const D d, V x) {
using T = TFromD<D>;
impl::LogImpl<T> impl;
constexpr bool kIsF32 = (sizeof(T) == 4);
const V kLn2Hi = Set(d, kIsF32 ? static_cast<T>(0.69313812256f)
: static_cast<T>(0.693147180369123816490));
const V kLn2Lo = Set(d, kIsF32 ? static_cast<T>(9.0580006145e-6f)
: static_cast<T>(1.90821492927058770002e-10));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kMinNormal = Set(d, kIsF32 ? static_cast<T>(1.175494351e-38f)
: static_cast<T>(2.2250738585072014e-308));
const V kScale = Set(d, kIsF32 ? static_cast<T>(3.355443200e+7f)
: static_cast<T>(1.8014398509481984e+16));
using TI = MakeSigned<T>;
const Rebind<TI, D> di;
using VI = decltype(Zero(di));
const VI kLowerBits = Set(di, kIsF32 ? static_cast<TI>(0x00000000L)
: static_cast<TI>(0xFFFFFFFFLL));
const VI kMagic = Set(di, kIsF32 ? static_cast<TI>(0x3F3504F3L)
: static_cast<TI>(0x3FE6A09E00000000LL));
const VI kExpMask = Set(di, kIsF32 ? static_cast<TI>(0x3F800000L)
: static_cast<TI>(0x3FF0000000000000LL));
const VI kExpScale =
Set(di, kIsF32 ? static_cast<TI>(-25) : static_cast<TI>(-54));
const VI kManMask = Set(di, kIsF32 ? static_cast<TI>(0x7FFFFFL)
: static_cast<TI>(0xFFFFF00000000LL));
VI exp_bits;
V exp;
if (kAllowSubnormals == true) {
const auto is_denormal = Lt(x, kMinNormal);
x = IfThenElse(is_denormal, Mul(x, kScale), x);
exp_bits = Add(BitCast(di, x), Sub(kExpMask, kMagic));
const VI exp_scale =
BitCast(di, IfThenElseZero(is_denormal, BitCast(d, kExpScale)));
exp = ConvertTo(
d, Add(exp_scale, impl.Log2p1NoSubnormal(d, BitCast(d, exp_bits))));
} else {
exp_bits = Add(BitCast(di, x), Sub(kExpMask, kMagic));
exp = ConvertTo(d, impl.Log2p1NoSubnormal(d, BitCast(d, exp_bits)));
}
const V y = Or(And(x, BitCast(d, kLowerBits)),
BitCast(d, Add(And(exp_bits, kManMask), kMagic)));
const V ym1 = Sub(y, kOne);
const V z = Div(ym1, Add(y, kOne));
return MulSub(
exp, kLn2Hi,
Sub(MulSub(z, Sub(ym1, impl.LogPoly(d, z)), Mul(exp, kLn2Lo)), ym1));
}
}
template <class D, class V>
inline __attribute__((always_inline)) V Acos(const D d, V x) {
using T = TFromD<D>;
const V kZero = Zero(d);
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kPi = Set(d, static_cast<T>(+3.14159265358979323846264));
const V kPiOverTwo = Set(d, static_cast<T>(+1.57079632679489661923132169));
const V sign_x = And(SignBit(d), x);
const V abs_x = Xor(x, sign_x);
const auto mask = Lt(abs_x, kHalf);
const V yy =
IfThenElse(mask, Mul(abs_x, abs_x), NegMulAdd(abs_x, kHalf, kHalf));
const V y = IfThenElse(mask, abs_x, Sqrt(yy));
impl::AsinImpl<T> impl;
const V t = Mul(impl.AsinPoly(d, yy, y), Mul(y, yy));
const V t_plus_y = Add(t, y);
const V z =
IfThenElse(mask, Sub(kPiOverTwo, Add(Xor(y, sign_x), Xor(t, sign_x))),
Add(t_plus_y, t_plus_y));
return IfThenElse(Or(mask, Ge(x, kZero)), z, Sub(kPi, z));
}
template <class D, class V>
inline __attribute__((always_inline)) V Acosh(const D d, V x) {
using T = TFromD<D>;
const V kLarge = Set(d, static_cast<T>(268435456.0));
const V kLog2 = Set(d, static_cast<T>(0.693147180559945286227));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kTwo = Set(d, static_cast<T>(+2.0));
const auto is_x_large = Gt(x, kLarge);
const auto is_x_gt_2 = Gt(x, kTwo);
const V x_minus_1 = Sub(x, kOne);
const V y0 = MulSub(kTwo, x, Div(kOne, Add(Sqrt(MulSub(x, x, kOne)), x)));
const V y1 =
Add(Sqrt(MulAdd(x_minus_1, kTwo, Mul(x_minus_1, x_minus_1))), x_minus_1);
const V y2 =
IfThenElse(is_x_gt_2, IfThenElse(is_x_large, x, y0), Add(y1, kOne));
const V z = impl::Log<D, V, false>(d, y2);
const auto is_pole = Eq(y2, kOne);
const auto divisor = Sub(IfThenZeroElse(is_pole, y2), kOne);
return Add(IfThenElse(is_x_gt_2, z,
IfThenElse(is_pole, y1, Div(Mul(z, y1), divisor))),
IfThenElseZero(is_x_large, kLog2));
}
template <class D, class V>
inline __attribute__((always_inline)) V Asin(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kTwo = Set(d, static_cast<T>(+2.0));
const V kPiOverTwo = Set(d, static_cast<T>(+1.57079632679489661923132169));
const V sign_x = And(SignBit(d), x);
const V abs_x = Xor(x, sign_x);
const auto mask = Lt(abs_x, kHalf);
const V yy =
IfThenElse(mask, Mul(abs_x, abs_x), NegMulAdd(abs_x, kHalf, kHalf));
const V y = IfThenElse(mask, abs_x, Sqrt(yy));
impl::AsinImpl<T> impl;
const V z0 = MulAdd(impl.AsinPoly(d, yy, y), Mul(yy, y), y);
const V z1 = NegMulAdd(z0, kTwo, kPiOverTwo);
return Or(IfThenElse(mask, z0, z1), sign_x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Asinh(const D d, V x) {
using T = TFromD<D>;
const V kSmall = Set(d, static_cast<T>(1.0 / 268435456.0));
const V kLarge = Set(d, static_cast<T>(268435456.0));
const V kLog2 = Set(d, static_cast<T>(0.693147180559945286227));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kTwo = Set(d, static_cast<T>(+2.0));
const V sign_x = And(SignBit(d), x);
const V abs_x = Xor(x, sign_x);
const auto is_x_large = Gt(abs_x, kLarge);
const auto is_x_lt_2 = Lt(abs_x, kTwo);
const V x2 = Mul(x, x);
const V sqrt_x2_plus_1 = Sqrt(Add(x2, kOne));
const V y0 = MulAdd(abs_x, kTwo, Div(kOne, Add(sqrt_x2_plus_1, abs_x)));
const V y1 = Add(Div(x2, Add(sqrt_x2_plus_1, kOne)), abs_x);
const V y2 =
IfThenElse(is_x_lt_2, Add(y1, kOne), IfThenElse(is_x_large, abs_x, y0));
const V z = impl::Log<D, V, false>(d, y2);
const auto is_pole = Eq(y2, kOne);
const auto divisor = Sub(IfThenZeroElse(is_pole, y2), kOne);
const auto large = IfThenElse(is_pole, y1, Div(Mul(z, y1), divisor));
const V y = IfThenElse(Lt(abs_x, kSmall), x, large);
return Or(Add(IfThenElse(is_x_lt_2, y, z), IfThenElseZero(is_x_large, kLog2)),
sign_x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Atan(const D d, V x) {
using T = TFromD<D>;
const V kOne = Set(d, static_cast<T>(+1.0));
const V kPiOverTwo = Set(d, static_cast<T>(+1.57079632679489661923132169));
const V sign = And(SignBit(d), x);
const V abs_x = Xor(x, sign);
const auto mask = Gt(abs_x, kOne);
impl::AtanImpl<T> impl;
const auto divisor = IfThenElse(mask, abs_x, kOne);
const V y = impl.AtanPoly(d, IfThenElse(mask, Div(kOne, divisor), abs_x));
return Or(IfThenElse(mask, Sub(kPiOverTwo, y), y), sign);
}
template <class D, class V>
inline __attribute__((always_inline)) V Atanh(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kOne = Set(d, static_cast<T>(+1.0));
const V sign = And(SignBit(d), x);
const V abs_x = Xor(x, sign);
return Mul(Log1p(d, Div(Add(abs_x, abs_x), Sub(kOne, abs_x))),
Xor(kHalf, sign));
}
template <class D, class V>
inline __attribute__((always_inline)) V Cos(const D d, V x) {
using T = TFromD<D>;
impl::CosSinImpl<T> impl;
const V kOneOverPi = Set(d, static_cast<T>(0.31830988618379067153));
const Rebind<int32_t, D> di32;
using VI32 = decltype(Zero(di32));
const VI32 kOne = Set(di32, 1);
const V y = Abs(x);
const VI32 q = Add(ShiftLeft<1>(impl.ToInt32(d, Mul(y, kOneOverPi))), kOne);
return impl.Poly(
d, Xor(impl.CosReduce(d, y, q), impl.CosSignFromQuadrant(d, q)));
}
template <class D, class V>
inline __attribute__((always_inline)) V Exp(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kLowerBound =
Set(d, static_cast<T>((sizeof(T) == 4 ? -104.0 : -1000.0)));
const V kNegZero = Set(d, static_cast<T>(-0.0));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kOneOverLog2 = Set(d, static_cast<T>(+1.442695040888963407359924681));
impl::ExpImpl<T> impl;
const auto q =
impl.ToInt32(d, MulAdd(x, kOneOverLog2, Or(kHalf, And(x, kNegZero))));
const V y = impl.LoadExpShortRange(
d, Add(impl.ExpPoly(d, impl.ExpReduce(d, x, q)), kOne), q);
return IfThenElseZero(Ge(x, kLowerBound), y);
}
template <class D, class V>
inline __attribute__((always_inline)) V Expm1(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kLowerBound =
Set(d, static_cast<T>((sizeof(T) == 4 ? -104.0 : -1000.0)));
const V kLn2Over2 = Set(d, static_cast<T>(+0.346573590279972654708616));
const V kNegOne = Set(d, static_cast<T>(-1.0));
const V kNegZero = Set(d, static_cast<T>(-0.0));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kOneOverLog2 = Set(d, static_cast<T>(+1.442695040888963407359924681));
impl::ExpImpl<T> impl;
const auto q =
impl.ToInt32(d, MulAdd(x, kOneOverLog2, Or(kHalf, And(x, kNegZero))));
const V y = impl.ExpPoly(d, impl.ExpReduce(d, x, q));
const V z = IfThenElse(Lt(Abs(x), kLn2Over2), y,
Sub(impl.LoadExpShortRange(d, Add(y, kOne), q), kOne));
return IfThenElse(Lt(x, kLowerBound), kNegOne, z);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log(const D d, V x) {
return impl::Log<D, V, true>(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log10(const D d, V x) {
using T = TFromD<D>;
return Mul(Log(d, x), Set(d, static_cast<T>(0.4342944819032518276511)));
}
template <class D, class V>
inline __attribute__((always_inline)) V Log1p(const D d, V x) {
using T = TFromD<D>;
const V kOne = Set(d, static_cast<T>(+1.0));
const V y = Add(x, kOne);
const auto is_pole = Eq(y, kOne);
const auto divisor = Sub(IfThenZeroElse(is_pole, y), kOne);
const auto non_pole =
Mul(impl::Log<D, V, false>(d, y), Div(x, divisor));
return IfThenElse(is_pole, x, non_pole);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log2(const D d, V x) {
using T = TFromD<D>;
return Mul(Log(d, x), Set(d, static_cast<T>(1.44269504088896340735992)));
}
template <class D, class V>
inline __attribute__((always_inline)) V Sin(const D d, V x) {
using T = TFromD<D>;
impl::CosSinImpl<T> impl;
const V kOneOverPi = Set(d, static_cast<T>(0.31830988618379067153));
const V kHalf = Set(d, static_cast<T>(0.5));
const Rebind<int32_t, D> di32;
using VI32 = decltype(Zero(di32));
const V abs_x = Abs(x);
const V sign_x = Xor(abs_x, x);
const VI32 q = impl.ToInt32(d, MulAdd(abs_x, kOneOverPi, kHalf));
return impl.Poly(d, Xor(impl.SinReduce(d, abs_x, q),
Xor(impl.SinSignFromQuadrant(d, q), sign_x)));
}
template <class D, class V>
inline __attribute__((always_inline)) V Sinh(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kTwo = Set(d, static_cast<T>(+2.0));
const V sign = And(SignBit(d), x);
const V abs_x = Xor(x, sign);
const V y = Expm1(d, abs_x);
const V z = Mul(Div(Add(y, kTwo), Add(y, kOne)), Mul(y, kHalf));
return Xor(z, sign);
}
template <class D, class V>
inline __attribute__((always_inline)) V Tanh(const D d, V x) {
using T = TFromD<D>;
const V kLimit = Set(d, static_cast<T>(18.714973875));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kTwo = Set(d, static_cast<T>(+2.0));
const V sign = And(SignBit(d), x);
const V abs_x = Xor(x, sign);
const V y = Expm1(d, Mul(abs_x, kTwo));
const V z = IfThenElse(Gt(abs_x, kLimit), kOne, Div(y, Add(y, kTwo)));
return Xor(z, sign);
}
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __detail
{
template<typename _Tp>
inline void
__return_temporary_buffer(_Tp* __p,
size_t __len __attribute__((__unused__)))
{
::operator delete(__p, __len * sizeof(_Tp));
}
}
template<typename _Tp>
[[__deprecated__]]
pair<_Tp*, ptrdiff_t>
get_temporary_buffer(ptrdiff_t __len) noexcept
{
const ptrdiff_t __max =
__gnu_cxx::__numeric_traits<ptrdiff_t>::__max / sizeof(_Tp);
if (__len > __max)
__len = __max;
while (__len > 0)
{
_Tp* __tmp = static_cast<_Tp*>(::operator new(__len * sizeof(_Tp),
std::nothrow));
if (__tmp != 0)
return std::pair<_Tp*, ptrdiff_t>(__tmp, __len);
__len = __len == 1 ? 0 : ((__len + 1) / 2);
}
return std::pair<_Tp*, ptrdiff_t>(static_cast<_Tp*>(0), 0);
}
template<typename _Tp>
inline void
return_temporary_buffer(_Tp* __p)
{ ::operator delete(__p); }
template<typename _ForwardIterator, typename _Tp>
class _Temporary_buffer
{
public:
typedef _Tp value_type;
typedef value_type* pointer;
typedef pointer iterator;
typedef ptrdiff_t size_type;
protected:
size_type _M_original_len;
size_type _M_len;
pointer _M_buffer;
public:
size_type
size() const
{ return _M_len; }
size_type
requested_size() const
{ return _M_original_len; }
iterator
begin()
{ return _M_buffer; }
iterator
end()
{ return _M_buffer + _M_len; }
_Temporary_buffer(_ForwardIterator __seed, size_type __original_len);
~_Temporary_buffer()
{
std::_Destroy(_M_buffer, _M_buffer + _M_len);
std::__detail::__return_temporary_buffer(_M_buffer, _M_len);
}
private:
_Temporary_buffer(const _Temporary_buffer&);
void
operator=(const _Temporary_buffer&);
};
template<bool>
struct __uninitialized_construct_buf_dispatch
{
template<typename _Pointer, typename _ForwardIterator>
static void
__ucr(_Pointer __first, _Pointer __last,
_ForwardIterator __seed)
{
if (__first == __last)
return;
_Pointer __cur = __first;
try
{
std::_Construct(std::__addressof(*__first),
std::move(*__seed));
_Pointer __prev = __cur;
++__cur;
for(; __cur != __last; ++__cur, ++__prev)
std::_Construct(std::__addressof(*__cur),
std::move(*__prev));
*__seed = std::move(*__prev);
}
catch(...)
{
std::_Destroy(__first, __cur);
throw;
}
}
};
template<>
struct __uninitialized_construct_buf_dispatch<true>
{
template<typename _Pointer, typename _ForwardIterator>
static void
__ucr(_Pointer, _Pointer, _ForwardIterator) { }
};
template<typename _Pointer, typename _ForwardIterator>
inline void
__uninitialized_construct_buf(_Pointer __first, _Pointer __last,
_ForwardIterator __seed)
{
typedef typename std::iterator_traits<_Pointer>::value_type
_ValueType;
std::__uninitialized_construct_buf_dispatch<
__has_trivial_constructor(_ValueType)>::
__ucr(__first, __last, __seed);
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _ForwardIterator, typename _Tp>
_Temporary_buffer<_ForwardIterator, _Tp>::
_Temporary_buffer(_ForwardIterator __seed, size_type __original_len)
: _M_original_len(__original_len), _M_len(0), _M_buffer(0)
{
std::pair<pointer, size_type> __p(
std::get_temporary_buffer<value_type>(_M_original_len));
if (__p.first)
{
try
{
std::__uninitialized_construct_buf(__p.first, __p.first + __p.second,
__seed);
_M_buffer = __p.first;
_M_len = __p.second;
}
catch(...)
{
std::__detail::__return_temporary_buffer(__p.first, __p.second);
throw;
}
}
}
#pragma GCC diagnostic pop
}
namespace std __attribute__ ((__visibility__ ("default")))
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template <class _OutputIterator, class _Tp>
class [[__deprecated__]] raw_storage_iterator
: public iterator<output_iterator_tag, void, void, void, void>
{
protected:
_OutputIterator _M_iter;
public:
explicit
raw_storage_iterator(_OutputIterator __x)
: _M_iter(__x) {}
raw_storage_iterator&
operator*() { return *this; }
raw_storage_iterator&
operator=(const _Tp& __element)
{
std::_Construct(std::__addressof(*_M_iter), __element);
return *this;
}
raw_storage_iterator&
operator=(_Tp&& __element)
{
std::_Construct(std::__addressof(*_M_iter), std::move(__element));
return *this;
}
raw_storage_iterator&
operator++()
{
++_M_iter;
return *this;
}
raw_storage_iterator
operator++(int)
{
raw_storage_iterator __tmp = *this;
++_M_iter;
return __tmp;
}
_OutputIterator base() const { return _M_iter; }
};
#pragma GCC diagnostic pop
}
namespace std __attribute__ ((__visibility__ ("default")))
{
inline void*
align(size_t __align, size_t __size, void*& __ptr, size_t& __space) noexcept
{
if (__space < __size)
return nullptr;
const auto __intptr = reinterpret_cast<uintptr_t>(__ptr);
const auto __aligned = (__intptr - 1u + __align) & -__align;
const auto __diff = __aligned - __intptr;
if (__diff > (__space - __size))
return nullptr;
else
{
__space -= __diff;
return __ptr = reinterpret_cast<void*>(__aligned);
}
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename> class auto_ptr;
#pragma GCC diagnostic pop
template<typename _Tp>
struct default_delete
{
constexpr default_delete() noexcept = default;
template<typename _Up,
typename = _Require<is_convertible<_Up*, _Tp*>>>
default_delete(const default_delete<_Up>&) noexcept { }
void
operator()(_Tp* __ptr) const
{
static_assert(!is_void<_Tp>::value,
"can't delete pointer to incomplete type");
static_assert(sizeof(_Tp)>0,
"can't delete pointer to incomplete type");
delete __ptr;
}
};
template<typename _Tp>
struct default_delete<_Tp[]>
{
public:
constexpr default_delete() noexcept = default;
template<typename _Up,
typename = _Require<is_convertible<_Up(*)[], _Tp(*)[]>>>
default_delete(const default_delete<_Up[]>&) noexcept { }
template<typename _Up>
typename enable_if<is_convertible<_Up(*)[], _Tp(*)[]>::value>::type
operator()(_Up* __ptr) const
{
static_assert(sizeof(_Tp)>0,
"can't delete pointer to incomplete type");
delete [] __ptr;
}
};
template <typename _Tp, typename _Dp>
class __uniq_ptr_impl
{
template <typename _Up, typename _Ep, typename = void>
struct _Ptr
{
using type = _Up*;
};
template <typename _Up, typename _Ep>
struct
_Ptr<_Up, _Ep, __void_t<typename remove_reference<_Ep>::type::pointer>>
{
using type = typename remove_reference<_Ep>::type::pointer;
};
public:
using _DeleterConstraint = enable_if<
__and_<__not_<is_pointer<_Dp>>,
is_default_constructible<_Dp>>::value>;
using pointer = typename _Ptr<_Tp, _Dp>::type;
static_assert( !is_rvalue_reference<_Dp>::value,
"unique_ptr's deleter type must be a function object type"
" or an lvalue reference type" );
__uniq_ptr_impl() = default;
__uniq_ptr_impl(pointer __p) : _M_t() { _M_ptr() = __p; }
template<typename _Del>
__uniq_ptr_impl(pointer __p, _Del&& __d)
: _M_t(__p, std::forward<_Del>(__d)) { }
__uniq_ptr_impl(__uniq_ptr_impl&& __u) noexcept
: _M_t(std::move(__u._M_t))
{ __u._M_ptr() = nullptr; }
__uniq_ptr_impl& operator=(__uniq_ptr_impl&& __u) noexcept
{
reset(__u.release());
_M_deleter() = std::forward<_Dp>(__u._M_deleter());
return *this;
}
pointer& _M_ptr() noexcept { return std::get<0>(_M_t); }
pointer _M_ptr() const noexcept { return std::get<0>(_M_t); }
_Dp& _M_deleter() noexcept { return std::get<1>(_M_t); }
const _Dp& _M_deleter() const noexcept { return std::get<1>(_M_t); }
void reset(pointer __p) noexcept
{
const pointer __old_p = _M_ptr();
_M_ptr() = __p;
if (__old_p)
_M_deleter()(__old_p);
}
pointer release() noexcept
{
pointer __p = _M_ptr();
_M_ptr() = nullptr;
return __p;
}
void
swap(__uniq_ptr_impl& __rhs) noexcept
{
using std::swap;
swap(this->_M_ptr(), __rhs._M_ptr());
swap(this->_M_deleter(), __rhs._M_deleter());
}
private:
tuple<pointer, _Dp> _M_t;
};
template <typename _Tp, typename _Dp,
bool = is_move_constructible<_Dp>::value,
bool = is_move_assignable<_Dp>::value>
struct __uniq_ptr_data : __uniq_ptr_impl<_Tp, _Dp>
{
using __uniq_ptr_impl<_Tp, _Dp>::__uniq_ptr_impl;
__uniq_ptr_data(__uniq_ptr_data&&) = default;
__uniq_ptr_data& operator=(__uniq_ptr_data&&) = default;
};
template <typename _Tp, typename _Dp>
struct __uniq_ptr_data<_Tp, _Dp, true, false> : __uniq_ptr_impl<_Tp, _Dp>
{
using __uniq_ptr_impl<_Tp, _Dp>::__uniq_ptr_impl;
__uniq_ptr_data(__uniq_ptr_data&&) = default;
__uniq_ptr_data& operator=(__uniq_ptr_data&&) = delete;
};
template <typename _Tp, typename _Dp>
struct __uniq_ptr_data<_Tp, _Dp, false, true> : __uniq_ptr_impl<_Tp, _Dp>
{
using __uniq_ptr_impl<_Tp, _Dp>::__uniq_ptr_impl;
__uniq_ptr_data(__uniq_ptr_data&&) = delete;
__uniq_ptr_data& operator=(__uniq_ptr_data&&) = default;
};
template <typename _Tp, typename _Dp>
struct __uniq_ptr_data<_Tp, _Dp, false, false> : __uniq_ptr_impl<_Tp, _Dp>
{
using __uniq_ptr_impl<_Tp, _Dp>::__uniq_ptr_impl;
__uniq_ptr_data(__uniq_ptr_data&&) = delete;
__uniq_ptr_data& operator=(__uniq_ptr_data&&) = delete;
};
template <typename _Tp, typename _Dp = default_delete<_Tp>>
class unique_ptr
{
template <typename _Up>
using _DeleterConstraint =
typename __uniq_ptr_impl<_Tp, _Up>::_DeleterConstraint::type;
__uniq_ptr_data<_Tp, _Dp> _M_t;
public:
using pointer = typename __uniq_ptr_impl<_Tp, _Dp>::pointer;
using element_type = _Tp;
using deleter_type = _Dp;
private:
template<typename _Up, typename _Ep>
using __safe_conversion_up = __and_<
is_convertible<typename unique_ptr<_Up, _Ep>::pointer, pointer>,
__not_<is_array<_Up>>
>;
public:
template<typename _Del = _Dp, typename = _DeleterConstraint<_Del>>
constexpr unique_ptr() noexcept
: _M_t()
{ }
template<typename _Del = _Dp, typename = _DeleterConstraint<_Del>>
explicit
unique_ptr(pointer __p) noexcept
: _M_t(__p)
{ }
template<typename _Del = deleter_type,
typename = _Require<is_copy_constructible<_Del>>>
unique_ptr(pointer __p, const deleter_type& __d) noexcept
: _M_t(__p, __d) { }
template<typename _Del = deleter_type,
typename = _Require<is_move_constructible<_Del>>>
unique_ptr(pointer __p,
__enable_if_t<!is_lvalue_reference<_Del>::value,
_Del&&> __d) noexcept
: _M_t(__p, std::move(__d))
{ }
template<typename _Del = deleter_type,
typename _DelUnref = typename remove_reference<_Del>::type>
unique_ptr(pointer,
__enable_if_t<is_lvalue_reference<_Del>::value,
_DelUnref&&>) = delete;
template<typename _Del = _Dp, typename = _DeleterConstraint<_Del>>
constexpr unique_ptr(nullptr_t) noexcept
: _M_t()
{ }
unique_ptr(unique_ptr&&) = default;
template<typename _Up, typename _Ep, typename = _Require<
__safe_conversion_up<_Up, _Ep>,
__conditional_t<is_reference<_Dp>::value,
is_same<_Ep, _Dp>,
is_convertible<_Ep, _Dp>>>>
unique_ptr(unique_ptr<_Up, _Ep>&& __u) noexcept
: _M_t(__u.release(), std::forward<_Ep>(__u.get_deleter()))
{ }
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Up, typename = _Require<
is_convertible<_Up*, _Tp*>, is_same<_Dp, default_delete<_Tp>>>>
unique_ptr(auto_ptr<_Up>&& __u) noexcept;
#pragma GCC diagnostic pop
~unique_ptr() noexcept
{
static_assert(__is_invocable<deleter_type&, pointer>::value,
"unique_ptr's deleter must be invocable with a pointer");
auto& __ptr = _M_t._M_ptr();
if (__ptr != nullptr)
get_deleter()(std::move(__ptr));
__ptr = pointer();
}
unique_ptr& operator=(unique_ptr&&) = default;
template<typename _Up, typename _Ep>
typename enable_if< __and_<
__safe_conversion_up<_Up, _Ep>,
is_assignable<deleter_type&, _Ep&&>
>::value,
unique_ptr&>::type
operator=(unique_ptr<_Up, _Ep>&& __u) noexcept
{
reset(__u.release());
get_deleter() = std::forward<_Ep>(__u.get_deleter());
return *this;
}
unique_ptr&
operator=(nullptr_t) noexcept
{
reset();
return *this;
}
typename add_lvalue_reference<element_type>::type
operator*() const noexcept(noexcept(*std::declval<pointer>()))
{
do { if (std::__is_constant_evaluated() && !bool(get() != pointer())) __builtin_unreachable(); } while (false);
return *get();
}
pointer
operator->() const noexcept
{
;
return get();
}
pointer
get() const noexcept
{ return _M_t._M_ptr(); }
deleter_type&
get_deleter() noexcept
{ return _M_t._M_deleter(); }
const deleter_type&
get_deleter() const noexcept
{ return _M_t._M_deleter(); }
explicit operator bool() const noexcept
{ return get() == pointer() ? false : true; }
pointer
release() noexcept
{ return _M_t.release(); }
void
reset(pointer __p = pointer()) noexcept
{
static_assert(__is_invocable<deleter_type&, pointer>::value,
"unique_ptr's deleter must be invocable with a pointer");
_M_t.reset(std::move(__p));
}
void
swap(unique_ptr& __u) noexcept
{
static_assert(__is_swappable<_Dp>::value, "deleter must be swappable");
_M_t.swap(__u._M_t);
}
unique_ptr(const unique_ptr&) = delete;
unique_ptr& operator=(const unique_ptr&) = delete;
};
template<typename _Tp, typename _Dp>
class unique_ptr<_Tp[], _Dp>
{
template <typename _Up>
using _DeleterConstraint =
typename __uniq_ptr_impl<_Tp, _Up>::_DeleterConstraint::type;
__uniq_ptr_data<_Tp, _Dp> _M_t;
template<typename _Up>
using __is_derived_Tp
= __and_< is_base_of<_Tp, _Up>,
__not_<is_same<__remove_cv_t<_Tp>, __remove_cv_t<_Up>>> >;
public:
using pointer = typename __uniq_ptr_impl<_Tp, _Dp>::pointer;
using element_type = _Tp;
using deleter_type = _Dp;
template<typename _Up, typename _Ep,
typename _UPtr = unique_ptr<_Up, _Ep>,
typename _UP_pointer = typename _UPtr::pointer,
typename _UP_element_type = typename _UPtr::element_type>
using __safe_conversion_up = __and_<
is_array<_Up>,
is_same<pointer, element_type*>,
is_same<_UP_pointer, _UP_element_type*>,
is_convertible<_UP_element_type(*)[], element_type(*)[]>
>;
template<typename _Up>
using __safe_conversion_raw = __and_<
__or_<__or_<is_same<_Up, pointer>,
is_same<_Up, nullptr_t>>,
__and_<is_pointer<_Up>,
is_same<pointer, element_type*>,
is_convertible<
typename remove_pointer<_Up>::type(*)[],
element_type(*)[]>
>
>
>;
template<typename _Del = _Dp, typename = _DeleterConstraint<_Del>>
constexpr unique_ptr() noexcept
: _M_t()
{ }
template<typename _Up,
typename _Vp = _Dp,
typename = _DeleterConstraint<_Vp>,
typename = typename enable_if<
__safe_conversion_raw<_Up>::value, bool>::type>
explicit
unique_ptr(_Up __p) noexcept
: _M_t(__p)
{ }
template<typename _Up, typename _Del = deleter_type,
typename = _Require<__safe_conversion_raw<_Up>,
is_copy_constructible<_Del>>>
unique_ptr(_Up __p, const deleter_type& __d) noexcept
: _M_t(__p, __d) { }
template<typename _Up, typename _Del = deleter_type,
typename = _Require<__safe_conversion_raw<_Up>,
is_move_constructible<_Del>>>
unique_ptr(_Up __p,
__enable_if_t<!is_lvalue_reference<_Del>::value,
_Del&&> __d) noexcept
: _M_t(std::move(__p), std::move(__d))
{ }
template<typename _Up, typename _Del = deleter_type,
typename _DelUnref = typename remove_reference<_Del>::type,
typename = _Require<__safe_conversion_raw<_Up>>>
unique_ptr(_Up,
__enable_if_t<is_lvalue_reference<_Del>::value,
_DelUnref&&>) = delete;
unique_ptr(unique_ptr&&) = default;
template<typename _Del = _Dp, typename = _DeleterConstraint<_Del>>
constexpr unique_ptr(nullptr_t) noexcept
: _M_t()
{ }
template<typename _Up, typename _Ep, typename = _Require<
__safe_conversion_up<_Up, _Ep>,
__conditional_t<is_reference<_Dp>::value,
is_same<_Ep, _Dp>,
is_convertible<_Ep, _Dp>>>>
unique_ptr(unique_ptr<_Up, _Ep>&& __u) noexcept
: _M_t(__u.release(), std::forward<_Ep>(__u.get_deleter()))
{ }
~unique_ptr()
{
auto& __ptr = _M_t._M_ptr();
if (__ptr != nullptr)
get_deleter()(__ptr);
__ptr = pointer();
}
unique_ptr&
operator=(unique_ptr&&) = default;
template<typename _Up, typename _Ep>
typename
enable_if<__and_<__safe_conversion_up<_Up, _Ep>,
is_assignable<deleter_type&, _Ep&&>
>::value,
unique_ptr&>::type
operator=(unique_ptr<_Up, _Ep>&& __u) noexcept
{
reset(__u.release());
get_deleter() = std::forward<_Ep>(__u.get_deleter());
return *this;
}
unique_ptr&
operator=(nullptr_t) noexcept
{
reset();
return *this;
}
typename std::add_lvalue_reference<element_type>::type
operator[](size_t __i) const
{
do { if (std::__is_constant_evaluated() && !bool(get() != pointer())) __builtin_unreachable(); } while (false);
return get()[__i];
}
pointer
get() const noexcept
{ return _M_t._M_ptr(); }
deleter_type&
get_deleter() noexcept
{ return _M_t._M_deleter(); }
const deleter_type&
get_deleter() const noexcept
{ return _M_t._M_deleter(); }
explicit operator bool() const noexcept
{ return get() == pointer() ? false : true; }
pointer
release() noexcept
{ return _M_t.release(); }
template <typename _Up,
typename = _Require<
__or_<is_same<_Up, pointer>,
__and_<is_same<pointer, element_type*>,
is_pointer<_Up>,
is_convertible<
typename remove_pointer<_Up>::type(*)[],
element_type(*)[]
>
>
>
>>
void
reset(_Up __p) noexcept
{ _M_t.reset(std::move(__p)); }
void reset(nullptr_t = nullptr) noexcept
{ reset(pointer()); }
void
swap(unique_ptr& __u) noexcept
{
static_assert(__is_swappable<_Dp>::value, "deleter must be swappable");
_M_t.swap(__u._M_t);
}
unique_ptr(const unique_ptr&) = delete;
unique_ptr& operator=(const unique_ptr&) = delete;
};
template<typename _Tp, typename _Dp>
inline
typename enable_if<__is_swappable<_Dp>::value>::type
swap(unique_ptr<_Tp, _Dp>& __x,
unique_ptr<_Tp, _Dp>& __y) noexcept
{ __x.swap(__y); }
template<typename _Tp, typename _Dp>
typename enable_if<!__is_swappable<_Dp>::value>::type
swap(unique_ptr<_Tp, _Dp>&,
unique_ptr<_Tp, _Dp>&) = delete;
template<typename _Tp, typename _Dp,
typename _Up, typename _Ep>
[[__nodiscard__]]
inline bool
operator==(const unique_ptr<_Tp, _Dp>& __x,
const unique_ptr<_Up, _Ep>& __y)
{ return __x.get() == __y.get(); }
template<typename _Tp, typename _Dp>
[[__nodiscard__]]
inline bool
operator==(const unique_ptr<_Tp, _Dp>& __x, nullptr_t) noexcept
{ return !__x; }
template<typename _Tp, typename _Dp>
[[__nodiscard__]]
inline bool
operator==(nullptr_t, const unique_ptr<_Tp, _Dp>& __x) noexcept
{ return !__x; }
template<typename _Tp, typename _Dp,
typename _Up, typename _Ep>
[[__nodiscard__]]
inline bool
operator!=(const unique_ptr<_Tp, _Dp>& __x,
const unique_ptr<_Up, _Ep>& __y)
{ return __x.get() != __y.get(); }
template<typename _Tp, typename _Dp>
[[__nodiscard__]]
inline bool
operator!=(const unique_ptr<_Tp, _Dp>& __x, nullptr_t) noexcept
{ return (bool)__x; }
template<typename _Tp, typename _Dp>
[[__nodiscard__]]
inline bool
operator!=(nullptr_t, const unique_ptr<_Tp, _Dp>& __x) noexcept
{ return (bool)__x; }
template<typename _Tp, typename _Dp,
typename _Up, typename _Ep>
[[__nodiscard__]]
inline bool
operator<(const unique_ptr<_Tp, _Dp>& __x,
const unique_ptr<_Up, _Ep>& __y)
{
typedef typename
std::common_type<typename unique_ptr<_Tp, _Dp>::pointer,
typename unique_ptr<_Up, _Ep>::pointer>::type _CT;
return std::less<_CT>()(__x.get(), __y.get());
}
template<typename _Tp, typename _Dp>
[[__nodiscard__]]
inline bool
operator<(const unique_ptr<_Tp, _Dp>& __x, nullptr_t)
{
return std::less<typename unique_ptr<_Tp, _Dp>::pointer>()(__x.get(),
nullptr);
}
template<typename _Tp, typename _Dp>
[[__nodiscard__]]
inline bool
operator<(nullptr_t, const unique_ptr<_Tp, _Dp>& __x)
{
return std::less<typename unique_ptr<_Tp, _Dp>::pointer>()(nullptr,
__x.get());
}
template<typename _Tp, typename _Dp,
typename _Up, typename _Ep>
[[__nodiscard__]]
inline bool
operator<=(const unique_ptr<_Tp, _Dp>& __x,
const unique_ptr<_Up, _Ep>& __y)
{ return !(__y < __x); }
template<typename _Tp, typename _Dp>
[[__nodiscard__]]
inline bool
operator<=(const unique_ptr<_Tp, _Dp>& __x, nullptr_t)
{ return !(nullptr < __x); }
template<typename _Tp, typename _Dp>
[[__nodiscard__]]
inline bool
operator<=(nullptr_t, const unique_ptr<_Tp, _Dp>& __x)
{ return !(__x < nullptr); }
template<typename _Tp, typename _Dp,
typename _Up, typename _Ep>
[[__nodiscard__]]
inline bool
operator>(const unique_ptr<_Tp, _Dp>& __x,
const unique_ptr<_Up, _Ep>& __y)
{ return (__y < __x); }
template<typename _Tp, typename _Dp>
[[__nodiscard__]]
inline bool
operator>(const unique_ptr<_Tp, _Dp>& __x, nullptr_t)
{
return std::less<typename unique_ptr<_Tp, _Dp>::pointer>()(nullptr,
__x.get());
}
template<typename _Tp, typename _Dp>
[[__nodiscard__]]
inline bool
operator>(nullptr_t, const unique_ptr<_Tp, _Dp>& __x)
{
return std::less<typename unique_ptr<_Tp, _Dp>::pointer>()(__x.get(),
nullptr);
}
template<typename _Tp, typename _Dp,
typename _Up, typename _Ep>
[[__nodiscard__]]
inline bool
operator>=(const unique_ptr<_Tp, _Dp>& __x,
const unique_ptr<_Up, _Ep>& __y)
{ return !(__x < __y); }
template<typename _Tp, typename _Dp>
[[__nodiscard__]]
inline bool
operator>=(const unique_ptr<_Tp, _Dp>& __x, nullptr_t)
{ return !(__x < nullptr); }
template<typename _Tp, typename _Dp>
[[__nodiscard__]] inline bool
operator>=(nullptr_t, const unique_ptr<_Tp, _Dp>& __x)
{ return !(nullptr < __x); }
template<typename _Up, typename _Ptr = typename _Up::pointer,
bool = __poison_hash<_Ptr>::__enable_hash_call>
struct __uniq_ptr_hash
: private __poison_hash<_Ptr>
{
size_t
operator()(const _Up& __u) const
noexcept(noexcept(std::declval<hash<_Ptr>>()(std::declval<_Ptr>())))
{ return hash<_Ptr>()(__u.get()); }
};
template<typename _Up, typename _Ptr>
struct __uniq_ptr_hash<_Up, _Ptr, false>
: private __poison_hash<_Ptr>
{ };
template<typename _Tp, typename _Dp>
struct hash<unique_ptr<_Tp, _Dp>>
: public __hash_base<size_t, unique_ptr<_Tp, _Dp>>,
public __uniq_ptr_hash<unique_ptr<_Tp, _Dp>>
{ };
namespace __detail
{
template<typename _Tp>
struct _MakeUniq
{ typedef unique_ptr<_Tp> __single_object; };
template<typename _Tp>
struct _MakeUniq<_Tp[]>
{ typedef unique_ptr<_Tp[]> __array; };
template<typename _Tp, size_t _Bound>
struct _MakeUniq<_Tp[_Bound]>
{ struct __invalid_type { }; };
template<typename _Tp>
using __unique_ptr_t = typename _MakeUniq<_Tp>::__single_object;
template<typename _Tp>
using __unique_ptr_array_t = typename _MakeUniq<_Tp>::__array;
template<typename _Tp>
using __invalid_make_unique_t = typename _MakeUniq<_Tp>::__invalid_type;
}
template<typename _Tp, typename... _Args>
inline __detail::__unique_ptr_t<_Tp>
make_unique(_Args&&... __args)
{ return unique_ptr<_Tp>(new _Tp(std::forward<_Args>(__args)...)); }
template<typename _Tp>
inline __detail::__unique_ptr_array_t<_Tp>
make_unique(size_t __num)
{ return unique_ptr<_Tp>(new remove_extent_t<_Tp>[__num]()); }
template<typename _Tp, typename... _Args>
__detail::__invalid_make_unique_t<_Tp>
make_unique(_Args&&...) = delete;
namespace __detail::__variant
{
template<typename> struct _Never_valueless_alt;
template<typename _Tp, typename _Del>
struct _Never_valueless_alt<std::unique_ptr<_Tp, _Del>>
: std::true_type
{ };
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<class _CharT>
struct char_traits;
template<> struct char_traits<char>;
template<> struct char_traits<wchar_t>;
template<> struct char_traits<char16_t>;
template<> struct char_traits<char32_t>;
namespace __cxx11 {
template<typename _CharT, typename _Traits = char_traits<_CharT>,
typename _Alloc = allocator<_CharT> >
class basic_string;
}
typedef basic_string<char> string;
typedef basic_string<wchar_t> wstring;
typedef basic_string<char16_t> u16string;
typedef basic_string<char32_t> u32string;
}
typedef unsigned int wint_t;
typedef __mbstate_t mbstate_t;
extern "C" {
struct tm;
extern wchar_t *wcscpy (wchar_t *__restrict __dest,
const wchar_t *__restrict __src)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern wchar_t *wcsncpy (wchar_t *__restrict __dest,
const wchar_t *__restrict __src, size_t __n)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern wchar_t *wcscat (wchar_t *__restrict __dest,
const wchar_t *__restrict __src)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern wchar_t *wcsncat (wchar_t *__restrict __dest,
const wchar_t *__restrict __src, size_t __n)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int wcscmp (const wchar_t *__s1, const wchar_t *__s2)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern int wcsncmp (const wchar_t *__s1, const wchar_t *__s2, size_t __n)
noexcept (true) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
extern int wcscasecmp (const wchar_t *__s1, const wchar_t *__s2) noexcept (true);
extern int wcsncasecmp (const wchar_t *__s1, const wchar_t *__s2,
size_t __n) noexcept (true);
extern int wcscasecmp_l (const wchar_t *__s1, const wchar_t *__s2,
locale_t __loc) noexcept (true);
extern int wcsncasecmp_l (const wchar_t *__s1, const wchar_t *__s2,
size_t __n, locale_t __loc) noexcept (true);
extern int wcscoll (const wchar_t *__s1, const wchar_t *__s2) noexcept (true);
extern size_t wcsxfrm (wchar_t *__restrict __s1,
const wchar_t *__restrict __s2, size_t __n) noexcept (true);
extern int wcscoll_l (const wchar_t *__s1, const wchar_t *__s2,
locale_t __loc) noexcept (true);
extern size_t wcsxfrm_l (wchar_t *__s1, const wchar_t *__s2,
size_t __n, locale_t __loc) noexcept (true);
extern wchar_t *wcsdup (const wchar_t *__s) noexcept (true)
__attribute__ ((__malloc__)) __attribute__ ((__malloc__ (__builtin_free, 1)));
extern "C++" wchar_t *wcschr (wchar_t *__wcs, wchar_t __wc)
noexcept (true) __asm ("wcschr") __attribute__ ((__pure__));
extern "C++" const wchar_t *wcschr (const wchar_t *__wcs, wchar_t __wc)
noexcept (true) __asm ("wcschr") __attribute__ ((__pure__));
extern "C++" wchar_t *wcsrchr (wchar_t *__wcs, wchar_t __wc)
noexcept (true) __asm ("wcsrchr") __attribute__ ((__pure__));
extern "C++" const wchar_t *wcsrchr (const wchar_t *__wcs, wchar_t __wc)
noexcept (true) __asm ("wcsrchr") __attribute__ ((__pure__));
extern wchar_t *wcschrnul (const wchar_t *__s, wchar_t __wc)
noexcept (true) __attribute__ ((__pure__));
extern size_t wcscspn (const wchar_t *__wcs, const wchar_t *__reject)
noexcept (true) __attribute__ ((__pure__));
extern size_t wcsspn (const wchar_t *__wcs, const wchar_t *__accept)
noexcept (true) __attribute__ ((__pure__));
extern "C++" wchar_t *wcspbrk (wchar_t *__wcs, const wchar_t *__accept)
noexcept (true) __asm ("wcspbrk") __attribute__ ((__pure__));
extern "C++" const wchar_t *wcspbrk (const wchar_t *__wcs,
const wchar_t *__accept)
noexcept (true) __asm ("wcspbrk") __attribute__ ((__pure__));
extern "C++" wchar_t *wcsstr (wchar_t *__haystack, const wchar_t *__needle)
noexcept (true) __asm ("wcsstr") __attribute__ ((__pure__));
extern "C++" const wchar_t *wcsstr (const wchar_t *__haystack,
const wchar_t *__needle)
noexcept (true) __asm ("wcsstr") __attribute__ ((__pure__));
extern wchar_t *wcstok (wchar_t *__restrict __s,
const wchar_t *__restrict __delim,
wchar_t **__restrict __ptr) noexcept (true);
extern size_t wcslen (const wchar_t *__s) noexcept (true) __attribute__ ((__pure__));
extern "C++" wchar_t *wcswcs (wchar_t *__haystack, const wchar_t *__needle)
noexcept (true) __asm ("wcswcs") __attribute__ ((__pure__));
extern "C++" const wchar_t *wcswcs (const wchar_t *__haystack,
const wchar_t *__needle)
noexcept (true) __asm ("wcswcs") __attribute__ ((__pure__));
extern size_t wcsnlen (const wchar_t *__s, size_t __maxlen)
noexcept (true) __attribute__ ((__pure__));
extern "C++" wchar_t *wmemchr (wchar_t *__s, wchar_t __c, size_t __n)
noexcept (true) __asm ("wmemchr") __attribute__ ((__pure__));
extern "C++" const wchar_t *wmemchr (const wchar_t *__s, wchar_t __c,
size_t __n)
noexcept (true) __asm ("wmemchr") __attribute__ ((__pure__));
extern int wmemcmp (const wchar_t *__s1, const wchar_t *__s2, size_t __n)
noexcept (true) __attribute__ ((__pure__));
extern wchar_t *wmemcpy (wchar_t *__restrict __s1,
const wchar_t *__restrict __s2, size_t __n) noexcept (true);
extern wchar_t *wmemmove (wchar_t *__s1, const wchar_t *__s2, size_t __n)
noexcept (true);
extern wchar_t *wmemset (wchar_t *__s, wchar_t __c, size_t __n) noexcept (true);
extern wchar_t *wmempcpy (wchar_t *__restrict __s1,
const wchar_t *__restrict __s2, size_t __n)
noexcept (true);
extern wint_t btowc (int __c) noexcept (true);
extern int wctob (wint_t __c) noexcept (true);
extern int mbsinit (const mbstate_t *__ps) noexcept (true) __attribute__ ((__pure__));
extern size_t mbrtowc (wchar_t *__restrict __pwc,
const char *__restrict __s, size_t __n,
mbstate_t *__restrict __p) noexcept (true);
extern size_t wcrtomb (char *__restrict __s, wchar_t __wc,
mbstate_t *__restrict __ps) noexcept (true);
extern size_t __mbrlen (const char *__restrict __s, size_t __n,
mbstate_t *__restrict __ps) noexcept (true);
extern size_t mbrlen (const char *__restrict __s, size_t __n,
mbstate_t *__restrict __ps) noexcept (true);
extern wint_t __btowc_alias (int __c) __asm ("btowc");
extern __inline __attribute__ ((__gnu_inline__)) wint_t
__attribute__ ((__leaf__)) btowc (int __c) noexcept (true)
{ return (__builtin_constant_p (__c) && __c >= '\0' && __c <= '\x7f'
? (wint_t) __c : __btowc_alias (__c)); }
extern int __wctob_alias (wint_t __c) __asm ("wctob");
extern __inline __attribute__ ((__gnu_inline__)) int
__attribute__ ((__leaf__)) wctob (wint_t __wc) noexcept (true)
{ return (__builtin_constant_p (__wc) && __wc >= L'\0' && __wc <= L'\x7f'
? (int) __wc : __wctob_alias (__wc)); }
extern __inline __attribute__ ((__gnu_inline__)) size_t
__attribute__ ((__leaf__)) mbrlen (const char *__restrict __s, size_t __n, mbstate_t *__restrict __ps) noexcept (true)
{ return (__ps != __null
? mbrtowc (__null, __s, __n, __ps) : __mbrlen (__s, __n, __null)); }
extern size_t mbsrtowcs (wchar_t *__restrict __dst,
const char **__restrict __src, size_t __len,
mbstate_t *__restrict __ps) noexcept (true);
extern size_t wcsrtombs (char *__restrict __dst,
const wchar_t **__restrict __src, size_t __len,
mbstate_t *__restrict __ps) noexcept (true);
extern size_t mbsnrtowcs (wchar_t *__restrict __dst,
const char **__restrict __src, size_t __nmc,
size_t __len, mbstate_t *__restrict __ps) noexcept (true);
extern size_t wcsnrtombs (char *__restrict __dst,
const wchar_t **__restrict __src,
size_t __nwc, size_t __len,
mbstate_t *__restrict __ps) noexcept (true);
extern int wcwidth (wchar_t __c) noexcept (true);
extern int wcswidth (const wchar_t *__s, size_t __n) noexcept (true);
extern double wcstod (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr) noexcept (true);
extern float wcstof (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr) noexcept (true);
extern long double wcstold (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr) noexcept (true);
extern _Float32 wcstof32 (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr) noexcept (true);
extern _Float64 wcstof64 (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr) noexcept (true);
extern _Float128 wcstof128 (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr) noexcept (true);
extern _Float32x wcstof32x (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr) noexcept (true);
extern _Float64x wcstof64x (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr) noexcept (true);
extern long int wcstol (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr, int __base) noexcept (true);
extern unsigned long int wcstoul (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr, int __base)
noexcept (true);
__extension__
extern long long int wcstoll (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr, int __base)
noexcept (true);
__extension__
extern unsigned long long int wcstoull (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr,
int __base) noexcept (true);
__extension__
extern long long int wcstoq (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr, int __base)
noexcept (true);
__extension__
extern unsigned long long int wcstouq (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr,
int __base) noexcept (true);
extern long int wcstol_l (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr, int __base,
locale_t __loc) noexcept (true);
extern unsigned long int wcstoul_l (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr,
int __base, locale_t __loc) noexcept (true);
__extension__
extern long long int wcstoll_l (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr,
int __base, locale_t __loc) noexcept (true);
__extension__
extern unsigned long long int wcstoull_l (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr,
int __base, locale_t __loc)
noexcept (true);
extern double wcstod_l (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr, locale_t __loc)
noexcept (true);
extern float wcstof_l (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr, locale_t __loc)
noexcept (true);
extern long double wcstold_l (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr,
locale_t __loc) noexcept (true);
extern _Float32 wcstof32_l (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr,
locale_t __loc) noexcept (true);
extern _Float64 wcstof64_l (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr,
locale_t __loc) noexcept (true);
extern _Float128 wcstof128_l (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr,
locale_t __loc) noexcept (true);
extern _Float32x wcstof32x_l (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr,
locale_t __loc) noexcept (true);
extern _Float64x wcstof64x_l (const wchar_t *__restrict __nptr,
wchar_t **__restrict __endptr,
locale_t __loc) noexcept (true);
extern wchar_t *wcpcpy (wchar_t *__restrict __dest,
const wchar_t *__restrict __src) noexcept (true);
extern wchar_t *wcpncpy (wchar_t *__restrict __dest,
const wchar_t *__restrict __src, size_t __n)
noexcept (true);
extern __FILE *open_wmemstream (wchar_t **__bufloc, size_t *__sizeloc) noexcept (true)
__attribute__ ((__malloc__)) __attribute__ ((__malloc__ (fclose, 1)));
extern int fwide (__FILE *__fp, int __mode) noexcept (true);
extern int fwprintf (__FILE *__restrict __stream,
const wchar_t *__restrict __format, ...)
;
extern int wprintf (const wchar_t *__restrict __format, ...)
;
extern int swprintf (wchar_t *__restrict __s, size_t __n,
const wchar_t *__restrict __format, ...)
noexcept (true) ;
extern int vfwprintf (__FILE *__restrict __s,
const wchar_t *__restrict __format,
__gnuc_va_list __arg)
;
extern int vwprintf (const wchar_t *__restrict __format,
__gnuc_va_list __arg)
;
extern int vswprintf (wchar_t *__restrict __s, size_t __n,
const wchar_t *__restrict __format,
__gnuc_va_list __arg)
noexcept (true) ;
extern int fwscanf (__FILE *__restrict __stream,
const wchar_t *__restrict __format, ...)
;
extern int wscanf (const wchar_t *__restrict __format, ...)
;
extern int swscanf (const wchar_t *__restrict __s,
const wchar_t *__restrict __format, ...)
noexcept (true) ;
extern int fwscanf (__FILE *__restrict __stream, const wchar_t *__restrict __format, ...) __asm__ ("" "__isoc99_fwscanf")
;
extern int wscanf (const wchar_t *__restrict __format, ...) __asm__ ("" "__isoc99_wscanf")
;
extern int swscanf (const wchar_t *__restrict __s, const wchar_t *__restrict __format, ...) noexcept (true) __asm__ ("" "__isoc99_swscanf")
;
extern int vfwscanf (__FILE *__restrict __s,
const wchar_t *__restrict __format,
__gnuc_va_list __arg)
;
extern int vwscanf (const wchar_t *__restrict __format,
__gnuc_va_list __arg)
;
extern int vswscanf (const wchar_t *__restrict __s,
const wchar_t *__restrict __format,
__gnuc_va_list __arg)
noexcept (true) ;
extern int vfwscanf (__FILE *__restrict __s, const wchar_t *__restrict __format, __gnuc_va_list __arg) __asm__ ("" "__isoc99_vfwscanf")
;
extern int vwscanf (const wchar_t *__restrict __format, __gnuc_va_list __arg) __asm__ ("" "__isoc99_vwscanf")
;
extern int vswscanf (const wchar_t *__restrict __s, const wchar_t *__restrict __format, __gnuc_va_list __arg) noexcept (true) __asm__ ("" "__isoc99_vswscanf")
;
extern wint_t fgetwc (__FILE *__stream);
extern wint_t getwc (__FILE *__stream);
extern wint_t getwchar (void);
extern wint_t fputwc (wchar_t __wc, __FILE *__stream);
extern wint_t putwc (wchar_t __wc, __FILE *__stream);
extern wint_t putwchar (wchar_t __wc);
extern wchar_t *fgetws (wchar_t *__restrict __ws, int __n,
__FILE *__restrict __stream);
extern int fputws (const wchar_t *__restrict __ws,
__FILE *__restrict __stream);
extern wint_t ungetwc (wint_t __wc, __FILE *__stream);
extern wint_t getwc_unlocked (__FILE *__stream);
extern wint_t getwchar_unlocked (void);
extern wint_t fgetwc_unlocked (__FILE *__stream);
extern wint_t fputwc_unlocked (wchar_t __wc, __FILE *__stream);
extern wint_t putwc_unlocked (wchar_t __wc, __FILE *__stream);
extern wint_t putwchar_unlocked (wchar_t __wc);
extern wchar_t *fgetws_unlocked (wchar_t *__restrict __ws, int __n,
__FILE *__restrict __stream);
extern int fputws_unlocked (const wchar_t *__restrict __ws,
__FILE *__restrict __stream);
extern size_t wcsftime (wchar_t *__restrict __s, size_t __maxsize,
const wchar_t *__restrict __format,
const struct tm *__restrict __tp) noexcept (true);
extern size_t wcsftime_l (wchar_t *__restrict __s, size_t __maxsize,
const wchar_t *__restrict __format,
const struct tm *__restrict __tp,
locale_t __loc) noexcept (true);
}
namespace std
{
using ::mbstate_t;
}
extern "C++"
{
namespace std __attribute__ ((__visibility__ ("default")))
{
using ::wint_t;
using ::btowc;
using ::fgetwc;
using ::fgetws;
using ::fputwc;
using ::fputws;
using ::fwide;
using ::fwprintf;
using ::fwscanf;
using ::getwc;
using ::getwchar;
using ::mbrlen;
using ::mbrtowc;
using ::mbsinit;
using ::mbsrtowcs;
using ::putwc;
using ::putwchar;
using ::swprintf;
using ::swscanf;
using ::ungetwc;
using ::vfwprintf;
using ::vfwscanf;
using ::vswprintf;
using ::vswscanf;
using ::vwprintf;
using ::vwscanf;
using ::wcrtomb;
using ::wcscat;
using ::wcscmp;
using ::wcscoll;
using ::wcscpy;
using ::wcscspn;
using ::wcsftime;
using ::wcslen;
using ::wcsncat;
using ::wcsncmp;
using ::wcsncpy;
using ::wcsrtombs;
using ::wcsspn;
using ::wcstod;
using ::wcstof;
using ::wcstok;
using ::wcstol;
using ::wcstoul;
using ::wcsxfrm;
using ::wctob;
using ::wmemcmp;
using ::wmemcpy;
using ::wmemmove;
using ::wmemset;
using ::wprintf;
using ::wscanf;
using ::wcschr;
using ::wcspbrk;
using ::wcsrchr;
using ::wcsstr;
using ::wmemchr;
}
}
namespace __gnu_cxx
{
using ::wcstold;
using ::wcstoll;
using ::wcstoull;
}
namespace std
{
using ::__gnu_cxx::wcstold;
using ::__gnu_cxx::wcstoll;
using ::__gnu_cxx::wcstoull;
}
namespace std
{
using std::wcstof;
using std::vfwscanf;
using std::vswscanf;
using std::vwscanf;
using std::wcstold;
using std::wcstoll;
using std::wcstoull;
}
namespace std __attribute__ ((__visibility__ ("default")))
{
typedef long long int streamoff;
typedef ptrdiff_t streamsize;
template<typename _StateT>
class fpos
{
private:
streamoff _M_off;
_StateT _M_state;
public:
fpos()
: _M_off(0), _M_state() { }
fpos(streamoff __off)
: _M_off(__off), _M_state() { }
fpos(const fpos&) = default;
fpos& operator=(const fpos&) = default;
~fpos() = default;
operator streamoff() const { return _M_off; }
void
state(_StateT __st)
{ _M_state = __st; }
_StateT
state() const
{ return _M_state; }
fpos&
operator+=(streamoff __off)
{
_M_off += __off;
return *this;
}
fpos&
operator-=(streamoff __off)
{
_M_off -= __off;
return *this;
}
fpos
operator+(streamoff __off) const
{
fpos __pos(*this);
__pos += __off;
return __pos;
}
fpos
operator-(streamoff __off) const
{
fpos __pos(*this);
__pos -= __off;
return __pos;
}
streamoff
operator-(const fpos& __other) const
{ return _M_off - __other._M_off; }
};
template<typename _StateT>
inline bool
operator==(const fpos<_StateT>& __lhs, const fpos<_StateT>& __rhs)
{ return streamoff(__lhs) == streamoff(__rhs); }
template<typename _StateT>
inline bool
operator!=(const fpos<_StateT>& __lhs, const fpos<_StateT>& __rhs)
{ return streamoff(__lhs) != streamoff(__rhs); }
typedef fpos<mbstate_t> streampos;
typedef fpos<mbstate_t> wstreampos;
typedef fpos<mbstate_t> u16streampos;
typedef fpos<mbstate_t> u32streampos;
}
namespace std __attribute__ ((__visibility__ ("default")))
{
class ios_base;
template<typename _CharT, typename _Traits = char_traits<_CharT> >
class basic_ios;
template<typename _CharT, typename _Traits = char_traits<_CharT> >
class basic_streambuf;
template<typename _CharT, typename _Traits = char_traits<_CharT> >
class basic_istream;
template<typename _CharT, typename _Traits = char_traits<_CharT> >
class basic_ostream;
template<typename _CharT, typename _Traits = char_traits<_CharT> >
class basic_iostream;
namespace __cxx11 {
template<typename _CharT, typename _Traits = char_traits<_CharT>,
typename _Alloc = allocator<_CharT> >
class basic_stringbuf;
template<typename _CharT, typename _Traits = char_traits<_CharT>,
typename _Alloc = allocator<_CharT> >
class basic_istringstream;
template<typename _CharT, typename _Traits = char_traits<_CharT>,
typename _Alloc = allocator<_CharT> >
class basic_ostringstream;
template<typename _CharT, typename _Traits = char_traits<_CharT>,
typename _Alloc = allocator<_CharT> >
class basic_stringstream;
}
template<typename _CharT, typename _Traits = char_traits<_CharT> >
class basic_filebuf;
template<typename _CharT, typename _Traits = char_traits<_CharT> >
class basic_ifstream;
template<typename _CharT, typename _Traits = char_traits<_CharT> >
class basic_ofstream;
template<typename _CharT, typename _Traits = char_traits<_CharT> >
class basic_fstream;
template<typename _CharT, typename _Traits = char_traits<_CharT> >
class istreambuf_iterator;
template<typename _CharT, typename _Traits = char_traits<_CharT> >
class ostreambuf_iterator;
typedef basic_ios<char> ios;
typedef basic_streambuf<char> streambuf;
typedef basic_istream<char> istream;
typedef basic_ostream<char> ostream;
typedef basic_iostream<char> iostream;
typedef basic_stringbuf<char> stringbuf;
typedef basic_istringstream<char> istringstream;
typedef basic_ostringstream<char> ostringstream;
typedef basic_stringstream<char> stringstream;
typedef basic_filebuf<char> filebuf;
typedef basic_ifstream<char> ifstream;
typedef basic_ofstream<char> ofstream;
typedef basic_fstream<char> fstream;
typedef basic_ios<wchar_t> wios;
typedef basic_streambuf<wchar_t> wstreambuf;
typedef basic_istream<wchar_t> wistream;
typedef basic_ostream<wchar_t> wostream;
typedef basic_iostream<wchar_t> wiostream;
typedef basic_stringbuf<wchar_t> wstringbuf;
typedef basic_istringstream<wchar_t> wistringstream;
typedef basic_ostringstream<wchar_t> wostringstream;
typedef basic_stringstream<wchar_t> wstringstream;
typedef basic_filebuf<wchar_t> wfilebuf;
typedef basic_ifstream<wchar_t> wifstream;
typedef basic_ofstream<wchar_t> wofstream;
typedef basic_fstream<wchar_t> wfstream;
}
#pragma GCC visibility push(default)
extern "C++" {
namespace __cxxabiv1
{
class __class_type_info;
}
namespace std
{
class type_info
{
public:
virtual ~type_info();
const char* name() const noexcept
{ return __name[0] == '*' ? __name + 1 : __name; }
bool before(const type_info& __arg) const noexcept;
bool operator==(const type_info& __arg) const noexcept;
bool operator!=(const type_info& __arg) const noexcept
{ return !operator==(__arg); }
size_t hash_code() const noexcept
{
return _Hash_bytes(name(), __builtin_strlen(name()),
static_cast<size_t>(0xc70f6907UL));
}
virtual bool __is_pointer_p() const;
virtual bool __is_function_p() const;
virtual bool __do_catch(const type_info *__thr_type, void **__thr_obj,
unsigned __outer) const;
virtual bool __do_upcast(const __cxxabiv1::__class_type_info *__target,
void **__obj_ptr) const;
protected:
const char *__name;
explicit type_info(const char *__n): __name(__n) { }
private:
type_info& operator=(const type_info&) = delete;
type_info(const type_info&) = delete;
};
inline bool
type_info::before(const type_info& __arg) const noexcept
{
if (__name[0] != '*' || __arg.__name[0] != '*')
return __builtin_strcmp (__name, __arg.__name) < 0;
return __name < __arg.__name;
}
inline bool
type_info::operator==(const type_info& __arg) const noexcept
{
if (std::__is_constant_evaluated())
return this == &__arg;
if (__name == __arg.__name)
return true;
return __name[0] != '*' && __builtin_strcmp (__name, __arg.name()) == 0;
}
class bad_cast : public exception
{
public:
bad_cast() noexcept { }
virtual ~bad_cast() noexcept;
virtual const char* what() const noexcept;
};
class bad_typeid : public exception
{
public:
bad_typeid () noexcept { }
virtual ~bad_typeid() noexcept;
virtual const char* what() const noexcept;
};
}
}
#pragma GCC visibility pop
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Alloc>
struct __allocated_ptr
{
using pointer = typename allocator_traits<_Alloc>::pointer;
using value_type = typename allocator_traits<_Alloc>::value_type;
__allocated_ptr(_Alloc& __a, pointer __ptr) noexcept
: _M_alloc(std::__addressof(__a)), _M_ptr(__ptr)
{ }
template<typename _Ptr,
typename _Req = _Require<is_same<_Ptr, value_type*>>>
__allocated_ptr(_Alloc& __a, _Ptr __ptr)
: _M_alloc(std::__addressof(__a)),
_M_ptr(pointer_traits<pointer>::pointer_to(*__ptr))
{ }
__allocated_ptr(__allocated_ptr&& __gd) noexcept
: _M_alloc(__gd._M_alloc), _M_ptr(__gd._M_ptr)
{ __gd._M_ptr = nullptr; }
~__allocated_ptr()
{
if (_M_ptr != nullptr)
std::allocator_traits<_Alloc>::deallocate(*_M_alloc, _M_ptr, 1);
}
__allocated_ptr&
operator=(std::nullptr_t) noexcept
{
_M_ptr = nullptr;
return *this;
}
value_type* get() { return std::__to_address(_M_ptr); }
private:
_Alloc* _M_alloc;
pointer _M_ptr;
};
template<typename _Alloc>
__allocated_ptr<_Alloc>
__allocate_guarded(_Alloc& __a)
{
return { __a, std::allocator_traits<_Alloc>::allocate(__a, 1) };
}
}
namespace __gnu_cxx
{
template<typename _Tp>
struct __aligned_membuf
{
struct _Tp2 { _Tp _M_t; };
alignas(__alignof__(_Tp2::_M_t)) unsigned char _M_storage[sizeof(_Tp)];
__aligned_membuf() = default;
__aligned_membuf(std::nullptr_t) { }
void*
_M_addr() noexcept
{ return static_cast<void*>(&_M_storage); }
const void*
_M_addr() const noexcept
{ return static_cast<const void*>(&_M_storage); }
_Tp*
_M_ptr() noexcept
{ return static_cast<_Tp*>(_M_addr()); }
const _Tp*
_M_ptr() const noexcept
{ return static_cast<const _Tp*>(_M_addr()); }
};
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Tp>
struct __aligned_buffer
: std::aligned_storage<sizeof(_Tp), __alignof__(_Tp)>
{
typename
std::aligned_storage<sizeof(_Tp), __alignof__(_Tp)>::type _M_storage;
__aligned_buffer() = default;
__aligned_buffer(std::nullptr_t) { }
void*
_M_addr() noexcept
{
return static_cast<void*>(&_M_storage);
}
const void*
_M_addr() const noexcept
{
return static_cast<const void*>(&_M_storage);
}
_Tp*
_M_ptr() noexcept
{ return static_cast<_Tp*>(_M_addr()); }
const _Tp*
_M_ptr() const noexcept
{ return static_cast<const _Tp*>(_M_addr()); }
};
#pragma GCC diagnostic pop
}
#pragma GCC visibility push(default)
struct sched_param
{
int sched_priority;
};
extern "C" {
extern int clone (int (*__fn) (void *__arg), void *__child_stack,
int __flags, void *__arg, ...) noexcept (true);
extern int unshare (int __flags) noexcept (true);
extern int sched_getcpu (void) noexcept (true);
extern int getcpu (unsigned int *, unsigned int *) noexcept (true);
extern int setns (int __fd, int __nstype) noexcept (true);
}
typedef unsigned long int __cpu_mask;
typedef struct
{
__cpu_mask __bits[1024 / (8 * sizeof (__cpu_mask))];
} cpu_set_t;
extern "C" {
extern int __sched_cpucount (size_t __setsize, const cpu_set_t *__setp)
noexcept (true);
extern cpu_set_t *__sched_cpualloc (size_t __count) noexcept (true) ;
extern void __sched_cpufree (cpu_set_t *__set) noexcept (true);
}
extern "C" {
extern int sched_setparam (__pid_t __pid, const struct sched_param *__param)
noexcept (true);
extern int sched_getparam (__pid_t __pid, struct sched_param *__param) noexcept (true);
extern int sched_setscheduler (__pid_t __pid, int __policy,
const struct sched_param *__param) noexcept (true);
extern int sched_getscheduler (__pid_t __pid) noexcept (true);
extern int sched_yield (void) noexcept (true);
extern int sched_get_priority_max (int __algorithm) noexcept (true);
extern int sched_get_priority_min (int __algorithm) noexcept (true);
extern int sched_rr_get_interval (__pid_t __pid, struct timespec *__t) noexcept (true);
extern int sched_setaffinity (__pid_t __pid, size_t __cpusetsize,
const cpu_set_t *__cpuset) noexcept (true);
extern int sched_getaffinity (__pid_t __pid, size_t __cpusetsize,
cpu_set_t *__cpuset) noexcept (true);
}
struct timex
{
unsigned int modes;
__syscall_slong_t offset;
__syscall_slong_t freq;
__syscall_slong_t maxerror;
__syscall_slong_t esterror;
int status;
__syscall_slong_t constant;
__syscall_slong_t precision;
__syscall_slong_t tolerance;
struct timeval time;
__syscall_slong_t tick;
__syscall_slong_t ppsfreq;
__syscall_slong_t jitter;
int shift;
__syscall_slong_t stabil;
__syscall_slong_t jitcnt;
__syscall_slong_t calcnt;
__syscall_slong_t errcnt;
__syscall_slong_t stbcnt;
int tai;
int :32; int :32; int :32; int :32;
int :32; int :32; int :32; int :32;
int :32; int :32; int :32;
};
extern "C" {
extern int clock_adjtime (__clockid_t __clock_id, struct timex *__utx) noexcept (true) __attribute__ ((__nonnull__ (2)));
}
struct tm
{
int tm_sec;
int tm_min;
int tm_hour;
int tm_mday;
int tm_mon;
int tm_year;
int tm_wday;
int tm_yday;
int tm_isdst;
long int tm_gmtoff;
const char *tm_zone;
};
struct itimerspec
{
struct timespec it_interval;
struct timespec it_value;
};
struct sigevent;
extern "C" {
extern clock_t clock (void) noexcept (true);
extern time_t time (time_t *__timer) noexcept (true);
extern double difftime (time_t __time1, time_t __time0)
noexcept (true) __attribute__ ((__const__));
extern time_t mktime (struct tm *__tp) noexcept (true);
extern size_t strftime (char *__restrict __s, size_t __maxsize,
const char *__restrict __format,
const struct tm *__restrict __tp) noexcept (true);
extern char *strptime (const char *__restrict __s,
const char *__restrict __fmt, struct tm *__tp)
noexcept (true);
extern size_t strftime_l (char *__restrict __s, size_t __maxsize,
const char *__restrict __format,
const struct tm *__restrict __tp,
locale_t __loc) noexcept (true);
extern char *strptime_l (const char *__restrict __s,
const char *__restrict __fmt, struct tm *__tp,
locale_t __loc) noexcept (true);
extern struct tm *gmtime (const time_t *__timer) noexcept (true);
extern struct tm *localtime (const time_t *__timer) noexcept (true);
extern struct tm *gmtime_r (const time_t *__restrict __timer,
struct tm *__restrict __tp) noexcept (true);
extern struct tm *localtime_r (const time_t *__restrict __timer,
struct tm *__restrict __tp) noexcept (true);
extern char *asctime (const struct tm *__tp) noexcept (true);
extern char *ctime (const time_t *__timer) noexcept (true);
extern char *asctime_r (const struct tm *__restrict __tp,
char *__restrict __buf) noexcept (true);
extern char *ctime_r (const time_t *__restrict __timer,
char *__restrict __buf) noexcept (true);
extern char *__tzname[2];
extern int __daylight;
extern long int __timezone;
extern char *tzname[2];
extern void tzset (void) noexcept (true);
extern int daylight;
extern long int timezone;
extern time_t timegm (struct tm *__tp) noexcept (true);
extern time_t timelocal (struct tm *__tp) noexcept (true);
extern int dysize (int __year) noexcept (true) __attribute__ ((__const__));
extern int nanosleep (const struct timespec *__requested_time,
struct timespec *__remaining);
extern int clock_getres (clockid_t __clock_id, struct timespec *__res) noexcept (true);
extern int clock_gettime (clockid_t __clock_id, struct timespec *__tp)
noexcept (true) __attribute__ ((__nonnull__ (2)));
extern int clock_settime (clockid_t __clock_id, const struct timespec *__tp)
noexcept (true) __attribute__ ((__nonnull__ (2)));
extern int clock_nanosleep (clockid_t __clock_id, int __flags,
const struct timespec *__req,
struct timespec *__rem);
extern int clock_getcpuclockid (pid_t __pid, clockid_t *__clock_id) noexcept (true);
extern int timer_create (clockid_t __clock_id,
struct sigevent *__restrict __evp,
timer_t *__restrict __timerid) noexcept (true);
extern int timer_delete (timer_t __timerid) noexcept (true);
extern int timer_settime (timer_t __timerid, int __flags,
const struct itimerspec *__restrict __value,
struct itimerspec *__restrict __ovalue) noexcept (true);
extern int timer_gettime (timer_t __timerid, struct itimerspec *__value)
noexcept (true);
extern int timer_getoverrun (timer_t __timerid) noexcept (true);
extern int timespec_get (struct timespec *__ts, int __base)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int timespec_getres (struct timespec *__ts, int __base)
noexcept (true);
extern int getdate_err;
extern struct tm *getdate (const char *__string);
extern int getdate_r (const char *__restrict __string,
struct tm *__restrict __resbufp);
}
typedef int __jmp_buf[6];
struct __jmp_buf_tag
{
__jmp_buf __jmpbuf;
int __mask_was_saved;
__sigset_t __saved_mask;
};
extern "C" {
extern long int __sysconf (int __name) noexcept (true);
}
enum
{
PTHREAD_CREATE_JOINABLE,
PTHREAD_CREATE_DETACHED
};
enum
{
PTHREAD_MUTEX_TIMED_NP,
PTHREAD_MUTEX_RECURSIVE_NP,
PTHREAD_MUTEX_ERRORCHECK_NP,
PTHREAD_MUTEX_ADAPTIVE_NP
,
PTHREAD_MUTEX_NORMAL = PTHREAD_MUTEX_TIMED_NP,
PTHREAD_MUTEX_RECURSIVE = PTHREAD_MUTEX_RECURSIVE_NP,
PTHREAD_MUTEX_ERRORCHECK = PTHREAD_MUTEX_ERRORCHECK_NP,
PTHREAD_MUTEX_DEFAULT = PTHREAD_MUTEX_NORMAL
, PTHREAD_MUTEX_FAST_NP = PTHREAD_MUTEX_TIMED_NP
};
enum
{
PTHREAD_MUTEX_STALLED,
PTHREAD_MUTEX_STALLED_NP = PTHREAD_MUTEX_STALLED,
PTHREAD_MUTEX_ROBUST,
PTHREAD_MUTEX_ROBUST_NP = PTHREAD_MUTEX_ROBUST
};
enum
{
PTHREAD_PRIO_NONE,
PTHREAD_PRIO_INHERIT,
PTHREAD_PRIO_PROTECT
};
enum
{
PTHREAD_RWLOCK_PREFER_READER_NP,
PTHREAD_RWLOCK_PREFER_WRITER_NP,
PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP,
PTHREAD_RWLOCK_DEFAULT_NP = PTHREAD_RWLOCK_PREFER_READER_NP
};
enum
{
PTHREAD_INHERIT_SCHED,
PTHREAD_EXPLICIT_SCHED
};
enum
{
PTHREAD_SCOPE_SYSTEM,
PTHREAD_SCOPE_PROCESS
};
enum
{
PTHREAD_PROCESS_PRIVATE,
PTHREAD_PROCESS_SHARED
};
struct _pthread_cleanup_buffer
{
void (*__routine) (void *);
void *__arg;
int __canceltype;
struct _pthread_cleanup_buffer *__prev;
};
enum
{
PTHREAD_CANCEL_ENABLE,
PTHREAD_CANCEL_DISABLE
};
enum
{
PTHREAD_CANCEL_DEFERRED,
PTHREAD_CANCEL_ASYNCHRONOUS
};
extern "C" {
extern int pthread_create (pthread_t *__restrict __newthread,
const pthread_attr_t *__restrict __attr,
void *(*__start_routine) (void *),
void *__restrict __arg) noexcept (true) __attribute__ ((__nonnull__ (1, 3)));
extern void pthread_exit (void *__retval) __attribute__ ((__noreturn__));
extern int pthread_join (pthread_t __th, void **__thread_return);
extern int pthread_tryjoin_np (pthread_t __th, void **__thread_return) noexcept (true);
extern int pthread_timedjoin_np (pthread_t __th, void **__thread_return,
const struct timespec *__abstime);
extern int pthread_clockjoin_np (pthread_t __th, void **__thread_return,
clockid_t __clockid,
const struct timespec *__abstime);
extern int pthread_detach (pthread_t __th) noexcept (true);
extern pthread_t pthread_self (void) noexcept (true) __attribute__ ((__const__));
extern int pthread_equal (pthread_t __thread1, pthread_t __thread2)
noexcept (true) __attribute__ ((__const__));
extern int pthread_attr_init (pthread_attr_t *__attr) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_attr_destroy (pthread_attr_t *__attr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_attr_getdetachstate (const pthread_attr_t *__attr,
int *__detachstate)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_attr_setdetachstate (pthread_attr_t *__attr,
int __detachstate)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_attr_getguardsize (const pthread_attr_t *__attr,
size_t *__guardsize)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_attr_setguardsize (pthread_attr_t *__attr,
size_t __guardsize)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_attr_getschedparam (const pthread_attr_t *__restrict __attr,
struct sched_param *__restrict __param)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_attr_setschedparam (pthread_attr_t *__restrict __attr,
const struct sched_param *__restrict
__param) noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_attr_getschedpolicy (const pthread_attr_t *__restrict
__attr, int *__restrict __policy)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_attr_setschedpolicy (pthread_attr_t *__attr, int __policy)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_attr_getinheritsched (const pthread_attr_t *__restrict
__attr, int *__restrict __inherit)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_attr_setinheritsched (pthread_attr_t *__attr,
int __inherit)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_attr_getscope (const pthread_attr_t *__restrict __attr,
int *__restrict __scope)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_attr_setscope (pthread_attr_t *__attr, int __scope)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_attr_getstackaddr (const pthread_attr_t *__restrict
__attr, void **__restrict __stackaddr)
noexcept (true) __attribute__ ((__nonnull__ (1, 2))) __attribute__ ((__deprecated__));
extern int pthread_attr_setstackaddr (pthread_attr_t *__attr,
void *__stackaddr)
noexcept (true) __attribute__ ((__nonnull__ (1))) __attribute__ ((__deprecated__));
extern int pthread_attr_getstacksize (const pthread_attr_t *__restrict
__attr, size_t *__restrict __stacksize)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_attr_setstacksize (pthread_attr_t *__attr,
size_t __stacksize)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_attr_getstack (const pthread_attr_t *__restrict __attr,
void **__restrict __stackaddr,
size_t *__restrict __stacksize)
noexcept (true) __attribute__ ((__nonnull__ (1, 2, 3)));
extern int pthread_attr_setstack (pthread_attr_t *__attr, void *__stackaddr,
size_t __stacksize) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_attr_setaffinity_np (pthread_attr_t *__attr,
size_t __cpusetsize,
const cpu_set_t *__cpuset)
noexcept (true) __attribute__ ((__nonnull__ (1, 3)));
extern int pthread_attr_getaffinity_np (const pthread_attr_t *__attr,
size_t __cpusetsize,
cpu_set_t *__cpuset)
noexcept (true) __attribute__ ((__nonnull__ (1, 3)));
extern int pthread_getattr_default_np (pthread_attr_t *__attr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_attr_setsigmask_np (pthread_attr_t *__attr,
const __sigset_t *sigmask);
extern int pthread_attr_getsigmask_np (const pthread_attr_t *__attr,
__sigset_t *sigmask);
extern int pthread_setattr_default_np (const pthread_attr_t *__attr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_getattr_np (pthread_t __th, pthread_attr_t *__attr)
noexcept (true) __attribute__ ((__nonnull__ (2)));
extern int pthread_setschedparam (pthread_t __target_thread, int __policy,
const struct sched_param *__param)
noexcept (true) __attribute__ ((__nonnull__ (3)));
extern int pthread_getschedparam (pthread_t __target_thread,
int *__restrict __policy,
struct sched_param *__restrict __param)
noexcept (true) __attribute__ ((__nonnull__ (2, 3)));
extern int pthread_setschedprio (pthread_t __target_thread, int __prio)
noexcept (true);
extern int pthread_getname_np (pthread_t __target_thread, char *__buf,
size_t __buflen)
noexcept (true) __attribute__ ((__nonnull__ (2)));
extern int pthread_setname_np (pthread_t __target_thread, const char *__name)
noexcept (true) __attribute__ ((__nonnull__ (2)));
extern int pthread_getconcurrency (void) noexcept (true);
extern int pthread_setconcurrency (int __level) noexcept (true);
extern int pthread_yield (void) noexcept (true);
extern int pthread_yield (void) noexcept (true) __asm__ ("" "sched_yield")
__attribute__ ((__deprecated__ ("pthread_yield is deprecated, use sched_yield instead")))
;
extern int pthread_setaffinity_np (pthread_t __th, size_t __cpusetsize,
const cpu_set_t *__cpuset)
noexcept (true) __attribute__ ((__nonnull__ (3)));
extern int pthread_getaffinity_np (pthread_t __th, size_t __cpusetsize,
cpu_set_t *__cpuset)
noexcept (true) __attribute__ ((__nonnull__ (3)));
extern int pthread_once (pthread_once_t *__once_control,
void (*__init_routine) (void)) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_setcancelstate (int __state, int *__oldstate);
extern int pthread_setcanceltype (int __type, int *__oldtype);
extern int pthread_cancel (pthread_t __th);
extern void pthread_testcancel (void);
struct __cancel_jmp_buf_tag
{
__jmp_buf __cancel_jmp_buf;
int __mask_was_saved;
};
typedef struct
{
struct __cancel_jmp_buf_tag __cancel_jmp_buf[1];
void *__pad[4];
} __pthread_unwind_buf_t __attribute__ ((__aligned__));
struct __pthread_cleanup_frame
{
void (*__cancel_routine) (void *);
void *__cancel_arg;
int __do_it;
int __cancel_type;
};
class __pthread_cleanup_class
{
void (*__cancel_routine) (void *);
void *__cancel_arg;
int __do_it;
int __cancel_type;
public:
__pthread_cleanup_class (void (*__fct) (void *), void *__arg)
: __cancel_routine (__fct), __cancel_arg (__arg), __do_it (1) { }
~__pthread_cleanup_class () { if (__do_it) __cancel_routine (__cancel_arg); }
void __setdoit (int __newval) { __do_it = __newval; }
void __defer () { pthread_setcanceltype (PTHREAD_CANCEL_DEFERRED,
&__cancel_type); }
void __restore () const { pthread_setcanceltype (__cancel_type, 0); }
};
extern int __sigsetjmp_cancel (struct __cancel_jmp_buf_tag __env[1], int __savemask) noexcept (true) __asm__ ("" "__sigsetjmp")
__attribute__ ((__returns_twice__));
extern int pthread_mutex_init (pthread_mutex_t *__mutex,
const pthread_mutexattr_t *__mutexattr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_mutex_destroy (pthread_mutex_t *__mutex)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_mutex_trylock (pthread_mutex_t *__mutex)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_mutex_lock (pthread_mutex_t *__mutex)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_mutex_timedlock (pthread_mutex_t *__restrict __mutex,
const struct timespec *__restrict
__abstime) noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_mutex_clocklock (pthread_mutex_t *__restrict __mutex,
clockid_t __clockid,
const struct timespec *__restrict
__abstime) noexcept (true) __attribute__ ((__nonnull__ (1, 3)));
extern int pthread_mutex_unlock (pthread_mutex_t *__mutex)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_mutex_getprioceiling (const pthread_mutex_t *
__restrict __mutex,
int *__restrict __prioceiling)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_mutex_setprioceiling (pthread_mutex_t *__restrict __mutex,
int __prioceiling,
int *__restrict __old_ceiling)
noexcept (true) __attribute__ ((__nonnull__ (1, 3)));
extern int pthread_mutex_consistent (pthread_mutex_t *__mutex)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_mutex_consistent_np (pthread_mutex_t *) noexcept (true) __asm__ ("" "pthread_mutex_consistent")
__attribute__ ((__nonnull__ (1)))
__attribute__ ((__deprecated__ ("pthread_mutex_consistent_np is deprecated, use pthread_mutex_consistent")))
;
extern int pthread_mutexattr_init (pthread_mutexattr_t *__attr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_mutexattr_destroy (pthread_mutexattr_t *__attr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_mutexattr_getpshared (const pthread_mutexattr_t *
__restrict __attr,
int *__restrict __pshared)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_mutexattr_setpshared (pthread_mutexattr_t *__attr,
int __pshared)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_mutexattr_gettype (const pthread_mutexattr_t *__restrict
__attr, int *__restrict __kind)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_mutexattr_settype (pthread_mutexattr_t *__attr, int __kind)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_mutexattr_getprotocol (const pthread_mutexattr_t *
__restrict __attr,
int *__restrict __protocol)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_mutexattr_setprotocol (pthread_mutexattr_t *__attr,
int __protocol)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_mutexattr_getprioceiling (const pthread_mutexattr_t *
__restrict __attr,
int *__restrict __prioceiling)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_mutexattr_setprioceiling (pthread_mutexattr_t *__attr,
int __prioceiling)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_mutexattr_getrobust (const pthread_mutexattr_t *__attr,
int *__robustness)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_mutexattr_getrobust_np (pthread_mutexattr_t *, int *) noexcept (true) __asm__ ("" "pthread_mutexattr_getrobust")
__attribute__ ((__nonnull__ (1)))
__attribute__ ((__deprecated__ ("pthread_mutexattr_getrobust_np is deprecated, use pthread_mutexattr_getrobust")))
;
extern int pthread_mutexattr_setrobust (pthread_mutexattr_t *__attr,
int __robustness)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_mutexattr_setrobust_np (pthread_mutexattr_t *, int) noexcept (true) __asm__ ("" "pthread_mutexattr_setrobust")
__attribute__ ((__nonnull__ (1)))
__attribute__ ((__deprecated__ ("pthread_mutexattr_setrobust_np is deprecated, use pthread_mutexattr_setrobust")))
;
extern int pthread_rwlock_init (pthread_rwlock_t *__restrict __rwlock,
const pthread_rwlockattr_t *__restrict
__attr) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_rwlock_destroy (pthread_rwlock_t *__rwlock)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_rwlock_rdlock (pthread_rwlock_t *__rwlock)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_rwlock_tryrdlock (pthread_rwlock_t *__rwlock)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_rwlock_timedrdlock (pthread_rwlock_t *__restrict __rwlock,
const struct timespec *__restrict
__abstime) noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_rwlock_clockrdlock (pthread_rwlock_t *__restrict __rwlock,
clockid_t __clockid,
const struct timespec *__restrict
__abstime) noexcept (true) __attribute__ ((__nonnull__ (1, 3)));
extern int pthread_rwlock_wrlock (pthread_rwlock_t *__rwlock)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_rwlock_trywrlock (pthread_rwlock_t *__rwlock)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_rwlock_timedwrlock (pthread_rwlock_t *__restrict __rwlock,
const struct timespec *__restrict
__abstime) noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_rwlock_clockwrlock (pthread_rwlock_t *__restrict __rwlock,
clockid_t __clockid,
const struct timespec *__restrict
__abstime) noexcept (true) __attribute__ ((__nonnull__ (1, 3)));
extern int pthread_rwlock_unlock (pthread_rwlock_t *__rwlock)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_rwlockattr_init (pthread_rwlockattr_t *__attr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_rwlockattr_destroy (pthread_rwlockattr_t *__attr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_rwlockattr_getpshared (const pthread_rwlockattr_t *
__restrict __attr,
int *__restrict __pshared)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_rwlockattr_setpshared (pthread_rwlockattr_t *__attr,
int __pshared)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_rwlockattr_getkind_np (const pthread_rwlockattr_t *
__restrict __attr,
int *__restrict __pref)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_rwlockattr_setkind_np (pthread_rwlockattr_t *__attr,
int __pref) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_cond_init (pthread_cond_t *__restrict __cond,
const pthread_condattr_t *__restrict __cond_attr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_cond_destroy (pthread_cond_t *__cond)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_cond_signal (pthread_cond_t *__cond)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_cond_broadcast (pthread_cond_t *__cond)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_cond_wait (pthread_cond_t *__restrict __cond,
pthread_mutex_t *__restrict __mutex)
__attribute__ ((__nonnull__ (1, 2)));
extern int pthread_cond_timedwait (pthread_cond_t *__restrict __cond,
pthread_mutex_t *__restrict __mutex,
const struct timespec *__restrict __abstime)
__attribute__ ((__nonnull__ (1, 2, 3)));
extern int pthread_cond_clockwait (pthread_cond_t *__restrict __cond,
pthread_mutex_t *__restrict __mutex,
__clockid_t __clock_id,
const struct timespec *__restrict __abstime)
__attribute__ ((__nonnull__ (1, 2, 4)));
extern int pthread_condattr_init (pthread_condattr_t *__attr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_condattr_destroy (pthread_condattr_t *__attr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_condattr_getpshared (const pthread_condattr_t *
__restrict __attr,
int *__restrict __pshared)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_condattr_setpshared (pthread_condattr_t *__attr,
int __pshared) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_condattr_getclock (const pthread_condattr_t *
__restrict __attr,
__clockid_t *__restrict __clock_id)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_condattr_setclock (pthread_condattr_t *__attr,
__clockid_t __clock_id)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_spin_init (pthread_spinlock_t *__lock, int __pshared)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_spin_destroy (pthread_spinlock_t *__lock)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_spin_lock (pthread_spinlock_t *__lock)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_spin_trylock (pthread_spinlock_t *__lock)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_spin_unlock (pthread_spinlock_t *__lock)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_barrier_init (pthread_barrier_t *__restrict __barrier,
const pthread_barrierattr_t *__restrict
__attr, unsigned int __count)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_barrier_destroy (pthread_barrier_t *__barrier)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_barrier_wait (pthread_barrier_t *__barrier)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_barrierattr_init (pthread_barrierattr_t *__attr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_barrierattr_destroy (pthread_barrierattr_t *__attr)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_barrierattr_getpshared (const pthread_barrierattr_t *
__restrict __attr,
int *__restrict __pshared)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int pthread_barrierattr_setpshared (pthread_barrierattr_t *__attr,
int __pshared)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_key_create (pthread_key_t *__key,
void (*__destr_function) (void *))
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int pthread_key_delete (pthread_key_t __key) noexcept (true);
extern void *pthread_getspecific (pthread_key_t __key) noexcept (true);
extern int pthread_setspecific (pthread_key_t __key,
const void *__pointer)
noexcept (true) __attribute__ ((__access__ (__none__, 2)));
extern int pthread_getcpuclockid (pthread_t __thread_id,
__clockid_t *__clock_id)
noexcept (true) __attribute__ ((__nonnull__ (2)));
extern int pthread_atfork (void (*__prepare) (void),
void (*__parent) (void),
void (*__child) (void)) noexcept (true);
extern __inline __attribute__ ((__gnu_inline__)) int
__attribute__ ((__leaf__)) pthread_equal (pthread_t __thread1, pthread_t __thread2) noexcept (true)
{
return __thread1 == __thread2;
}
}
typedef pthread_t __gthread_t;
typedef pthread_key_t __gthread_key_t;
typedef pthread_once_t __gthread_once_t;
typedef pthread_mutex_t __gthread_mutex_t;
typedef pthread_mutex_t __gthread_recursive_mutex_t;
typedef pthread_cond_t __gthread_cond_t;
typedef struct timespec __gthread_time_t;
static inline int
__gthread_active_p (void)
{
return 1;
}
static inline int
__gthread_create (__gthread_t *__threadid, void *(*__func) (void*),
void *__args)
{
return pthread_create (__threadid, __null, __func, __args);
}
static inline int
__gthread_join (__gthread_t __threadid, void **__value_ptr)
{
return pthread_join (__threadid, __value_ptr);
}
static inline int
__gthread_detach (__gthread_t __threadid)
{
return pthread_detach (__threadid);
}
static inline int
__gthread_equal (__gthread_t __t1, __gthread_t __t2)
{
return pthread_equal (__t1, __t2);
}
static inline __gthread_t
__gthread_self (void)
{
return pthread_self ();
}
static inline int
__gthread_yield (void)
{
return sched_yield ();
}
static inline int
__gthread_once (__gthread_once_t *__once, void (*__func) (void))
{
if (__gthread_active_p ())
return pthread_once (__once, __func);
else
return -1;
}
static inline int
__gthread_key_create (__gthread_key_t *__key, void (*__dtor) (void *))
{
return pthread_key_create (__key, __dtor);
}
static inline int
__gthread_key_delete (__gthread_key_t __key)
{
return pthread_key_delete (__key);
}
static inline void *
__gthread_getspecific (__gthread_key_t __key)
{
return pthread_getspecific (__key);
}
static inline int
__gthread_setspecific (__gthread_key_t __key, const void *__ptr)
{
return pthread_setspecific (__key, __ptr);
}
static inline void
__gthread_mutex_init_function (__gthread_mutex_t *__mutex)
{
if (__gthread_active_p ())
pthread_mutex_init (__mutex, __null);
}
static inline int
__gthread_mutex_destroy (__gthread_mutex_t *__mutex)
{
if (__gthread_active_p ())
return pthread_mutex_destroy (__mutex);
else
return 0;
}
static inline int
__gthread_mutex_lock (__gthread_mutex_t *__mutex)
{
if (__gthread_active_p ())
return pthread_mutex_lock (__mutex);
else
return 0;
}
static inline int
__gthread_mutex_trylock (__gthread_mutex_t *__mutex)
{
if (__gthread_active_p ())
return pthread_mutex_trylock (__mutex);
else
return 0;
}
static inline int
__gthread_mutex_timedlock (__gthread_mutex_t *__mutex,
const __gthread_time_t *__abs_timeout)
{
if (__gthread_active_p ())
return pthread_mutex_timedlock (__mutex, __abs_timeout);
else
return 0;
}
static inline int
__gthread_mutex_unlock (__gthread_mutex_t *__mutex)
{
if (__gthread_active_p ())
return pthread_mutex_unlock (__mutex);
else
return 0;
}
static inline int
__gthread_recursive_mutex_lock (__gthread_recursive_mutex_t *__mutex)
{
return __gthread_mutex_lock (__mutex);
}
static inline int
__gthread_recursive_mutex_trylock (__gthread_recursive_mutex_t *__mutex)
{
return __gthread_mutex_trylock (__mutex);
}
static inline int
__gthread_recursive_mutex_timedlock (__gthread_recursive_mutex_t *__mutex,
const __gthread_time_t *__abs_timeout)
{
return __gthread_mutex_timedlock (__mutex, __abs_timeout);
}
static inline int
__gthread_recursive_mutex_unlock (__gthread_recursive_mutex_t *__mutex)
{
return __gthread_mutex_unlock (__mutex);
}
static inline int
__gthread_recursive_mutex_destroy (__gthread_recursive_mutex_t *__mutex)
{
return __gthread_mutex_destroy (__mutex);
}
static inline int
__gthread_cond_broadcast (__gthread_cond_t *__cond)
{
return pthread_cond_broadcast (__cond);
}
static inline int
__gthread_cond_signal (__gthread_cond_t *__cond)
{
return pthread_cond_signal (__cond);
}
static inline int
__gthread_cond_wait (__gthread_cond_t *__cond, __gthread_mutex_t *__mutex)
{
return pthread_cond_wait (__cond, __mutex);
}
static inline int
__gthread_cond_timedwait (__gthread_cond_t *__cond, __gthread_mutex_t *__mutex,
const __gthread_time_t *__abs_timeout)
{
return pthread_cond_timedwait (__cond, __mutex, __abs_timeout);
}
static inline int
__gthread_cond_wait_recursive (__gthread_cond_t *__cond,
__gthread_recursive_mutex_t *__mutex)
{
return __gthread_cond_wait (__cond, __mutex);
}
static inline int
__gthread_cond_destroy (__gthread_cond_t* __cond)
{
return pthread_cond_destroy (__cond);
}
#pragma GCC visibility pop
typedef int _Atomic_word;
extern "C" {
extern char __libc_single_threaded;
}
namespace __gnu_cxx __attribute__ ((__visibility__ ("default")))
{
__attribute__((__always_inline__))
inline bool
__is_single_threaded() noexcept
{
return ::__libc_single_threaded;
}
inline _Atomic_word
__attribute__((__always_inline__))
__exchange_and_add(volatile _Atomic_word* __mem, int __val)
{ return __atomic_fetch_add(__mem, __val, 4); }
inline void
__attribute__((__always_inline__))
__atomic_add(volatile _Atomic_word* __mem, int __val)
{ __atomic_fetch_add(__mem, __val, 4); }
inline _Atomic_word
__attribute__((__always_inline__))
__exchange_and_add_single(_Atomic_word* __mem, int __val)
{
_Atomic_word __result = *__mem;
*__mem += __val;
return __result;
}
inline void
__attribute__((__always_inline__))
__atomic_add_single(_Atomic_word* __mem, int __val)
{ *__mem += __val; }
inline _Atomic_word
__attribute__ ((__always_inline__))
__exchange_and_add_dispatch(_Atomic_word* __mem, int __val)
{
if (__is_single_threaded())
return __exchange_and_add_single(__mem, __val);
else
return __exchange_and_add(__mem, __val);
}
inline void
__attribute__ ((__always_inline__))
__atomic_add_dispatch(_Atomic_word* __mem, int __val)
{
if (__is_single_threaded())
__atomic_add_single(__mem, __val);
else
__atomic_add(__mem, __val);
}
}
extern "C++" {
namespace std __attribute__ ((__visibility__ ("default")))
{
class bad_exception : public exception
{
public:
bad_exception() noexcept { }
virtual ~bad_exception() noexcept;
virtual const char*
what() const noexcept;
};
typedef void (*terminate_handler) ();
terminate_handler set_terminate(terminate_handler) noexcept;
terminate_handler get_terminate() noexcept;
void terminate() noexcept __attribute__ ((__noreturn__));
typedef void (*__attribute__ ((__deprecated__)) unexpected_handler) ();
__attribute__ ((__deprecated__))
unexpected_handler set_unexpected(unexpected_handler) noexcept;
__attribute__ ((__deprecated__))
unexpected_handler get_unexpected() noexcept;
__attribute__ ((__deprecated__))
void unexpected() __attribute__ ((__noreturn__));
__attribute__ ((__deprecated__ ("use '" "std::uncaught_exceptions()" "' instead")))
bool uncaught_exception() noexcept __attribute__ ((__pure__));
int uncaught_exceptions() noexcept __attribute__ ((__pure__));
}
namespace __gnu_cxx __attribute__ ((__visibility__ ("default")))
{
void __verbose_terminate_handler();
}
}
#pragma GCC visibility push(default)
namespace std
{
class type_info;
}
namespace __cxxabiv1
{
struct __cxa_refcounted_exception;
extern "C"
{
void*
__cxa_allocate_exception(size_t) noexcept;
void
__cxa_free_exception(void*) noexcept;
__cxa_refcounted_exception*
__cxa_init_primary_exception(void *__object, std::type_info *__tinfo,
void ( *__dest) (void *))
noexcept;
}
}
#pragma GCC visibility pop
extern "C++" {
namespace std __attribute__ ((__visibility__ ("default")))
{
class type_info;
namespace __exception_ptr
{
class exception_ptr;
}
using __exception_ptr::exception_ptr;
exception_ptr current_exception() noexcept;
template<typename _Ex>
exception_ptr make_exception_ptr(_Ex) noexcept;
void rethrow_exception(exception_ptr) __attribute__ ((__noreturn__));
namespace __exception_ptr
{
using std::rethrow_exception;
class exception_ptr
{
void* _M_exception_object;
explicit exception_ptr(void* __e) noexcept;
void _M_addref() noexcept;
void _M_release() noexcept;
void *_M_get() const noexcept __attribute__ ((__pure__));
friend exception_ptr std::current_exception() noexcept;
friend void std::rethrow_exception(exception_ptr);
template<typename _Ex>
friend exception_ptr std::make_exception_ptr(_Ex) noexcept;
public:
exception_ptr() noexcept;
exception_ptr(const exception_ptr&) noexcept;
exception_ptr(nullptr_t) noexcept
: _M_exception_object(nullptr)
{ }
exception_ptr(exception_ptr&& __o) noexcept
: _M_exception_object(__o._M_exception_object)
{ __o._M_exception_object = nullptr; }
exception_ptr&
operator=(const exception_ptr&) noexcept;
exception_ptr&
operator=(exception_ptr&& __o) noexcept
{
exception_ptr(static_cast<exception_ptr&&>(__o)).swap(*this);
return *this;
}
~exception_ptr() noexcept;
void
swap(exception_ptr&) noexcept;
explicit operator bool() const noexcept
{ return _M_exception_object; }
friend bool
operator==(const exception_ptr& __x, const exception_ptr& __y)
noexcept
{ return __x._M_exception_object == __y._M_exception_object; }
friend bool
operator!=(const exception_ptr& __x, const exception_ptr& __y)
noexcept
{ return __x._M_exception_object != __y._M_exception_object; }
const class std::type_info*
__cxa_exception_type() const noexcept
__attribute__ ((__pure__));
};
inline
exception_ptr::exception_ptr() noexcept
: _M_exception_object(0)
{ }
inline
exception_ptr::exception_ptr(const exception_ptr& __other)
noexcept
: _M_exception_object(__other._M_exception_object)
{
if (_M_exception_object)
_M_addref();
}
inline
exception_ptr::~exception_ptr() noexcept
{
if (_M_exception_object)
_M_release();
}
inline exception_ptr&
exception_ptr::operator=(const exception_ptr& __other) noexcept
{
exception_ptr(__other).swap(*this);
return *this;
}
inline void
exception_ptr::swap(exception_ptr &__other) noexcept
{
void *__tmp = _M_exception_object;
_M_exception_object = __other._M_exception_object;
__other._M_exception_object = __tmp;
}
inline void
swap(exception_ptr& __lhs, exception_ptr& __rhs)
{ __lhs.swap(__rhs); }
template<typename _Ex>
inline void
__dest_thunk(void* __x)
{ static_cast<_Ex*>(__x)->~_Ex(); }
}
using __exception_ptr::swap;
template<typename _Ex>
exception_ptr
make_exception_ptr(_Ex __ex) noexcept
{
using _Ex2 = typename decay<_Ex>::type;
void* __e = __cxxabiv1::__cxa_allocate_exception(sizeof(_Ex));
(void) __cxxabiv1::__cxa_init_primary_exception(
__e, const_cast<std::type_info*>(&typeid(_Ex)),
__exception_ptr::__dest_thunk<_Ex2>);
try
{
::new (__e) _Ex2(__ex);
return exception_ptr(__e);
}
catch(...)
{
__cxxabiv1::__cxa_free_exception(__e);
return current_exception();
}
}
}
}
extern "C++" {
namespace std __attribute__ ((__visibility__ ("default")))
{
class nested_exception
{
exception_ptr _M_ptr;
public:
nested_exception() noexcept : _M_ptr(current_exception()) { }
nested_exception(const nested_exception&) noexcept = default;
nested_exception& operator=(const nested_exception&) noexcept = default;
virtual ~nested_exception() noexcept;
[[noreturn]]
void
rethrow_nested() const
{
if (_M_ptr)
rethrow_exception(_M_ptr);
std::terminate();
}
exception_ptr
nested_ptr() const noexcept
{ return _M_ptr; }
};
template<typename _Except>
struct _Nested_exception : public _Except, public nested_exception
{
explicit _Nested_exception(const _Except& __ex)
: _Except(__ex)
{ }
explicit _Nested_exception(_Except&& __ex)
: _Except(static_cast<_Except&&>(__ex))
{ }
};
template<typename _Tp>
[[noreturn]]
inline void
throw_with_nested(_Tp&& __t)
{
using _Up = typename decay<_Tp>::type;
using _CopyConstructible
= __and_<is_copy_constructible<_Up>, is_move_constructible<_Up>>;
static_assert(_CopyConstructible::value,
"throw_with_nested argument must be CopyConstructible");
if constexpr (is_class_v<_Up>)
if constexpr (!is_final_v<_Up>)
if constexpr (!is_base_of_v<nested_exception, _Up>)
throw _Nested_exception<_Up>{std::forward<_Tp>(__t)};
throw std::forward<_Tp>(__t);
}
template<typename _Ex>
inline void
rethrow_if_nested(const _Ex& __ex)
{
const _Ex* __ptr = __builtin_addressof(__ex);
if constexpr (!is_polymorphic_v<_Ex>)
return;
else if constexpr (is_base_of_v<nested_exception, _Ex>
&& !is_convertible_v<_Ex*, nested_exception*>)
return;
else if (auto __ne_ptr = dynamic_cast<const nested_exception*>(__ptr))
__ne_ptr->rethrow_nested();
}
}
}
namespace __gnu_cxx __attribute__ ((__visibility__ ("default")))
{
enum _Lock_policy { _S_single, _S_mutex, _S_atomic };
inline const _Lock_policy __default_lock_policy =
_S_atomic;
class __concurrence_lock_error : public std::exception
{
public:
virtual char const*
what() const throw()
{ return "__gnu_cxx::__concurrence_lock_error"; }
};
class __concurrence_unlock_error : public std::exception
{
public:
virtual char const*
what() const throw()
{ return "__gnu_cxx::__concurrence_unlock_error"; }
};
class __concurrence_broadcast_error : public std::exception
{
public:
virtual char const*
what() const throw()
{ return "__gnu_cxx::__concurrence_broadcast_error"; }
};
class __concurrence_wait_error : public std::exception
{
public:
virtual char const*
what() const throw()
{ return "__gnu_cxx::__concurrence_wait_error"; }
};
inline void
__throw_concurrence_lock_error()
{ (throw (__concurrence_lock_error())); }
inline void
__throw_concurrence_unlock_error()
{ (throw (__concurrence_unlock_error())); }
inline void
__throw_concurrence_broadcast_error()
{ (throw (__concurrence_broadcast_error())); }
inline void
__throw_concurrence_wait_error()
{ (throw (__concurrence_wait_error())); }
class __mutex
{
private:
__gthread_mutex_t _M_mutex = { { 0, 0, 0, PTHREAD_MUTEX_TIMED_NP, 0, { { 0, 0 } } } };
__mutex(const __mutex&);
__mutex& operator=(const __mutex&);
public:
__mutex()
{
}
void lock()
{
if (__gthread_active_p())
{
if (__gthread_mutex_lock(&_M_mutex) != 0)
__throw_concurrence_lock_error();
}
}
void unlock()
{
if (__gthread_active_p())
{
if (__gthread_mutex_unlock(&_M_mutex) != 0)
__throw_concurrence_unlock_error();
}
}
__gthread_mutex_t* gthread_mutex(void)
{ return &_M_mutex; }
};
class __recursive_mutex
{
private:
__gthread_recursive_mutex_t _M_mutex = { { 0, 0, 0, PTHREAD_MUTEX_RECURSIVE_NP, 0, { { 0, 0 } } } };
__recursive_mutex(const __recursive_mutex&);
__recursive_mutex& operator=(const __recursive_mutex&);
public:
__recursive_mutex()
{
}
void lock()
{
if (__gthread_active_p())
{
if (__gthread_recursive_mutex_lock(&_M_mutex) != 0)
__throw_concurrence_lock_error();
}
}
void unlock()
{
if (__gthread_active_p())
{
if (__gthread_recursive_mutex_unlock(&_M_mutex) != 0)
__throw_concurrence_unlock_error();
}
}
__gthread_recursive_mutex_t* gthread_recursive_mutex(void)
{ return &_M_mutex; }
};
class __scoped_lock
{
public:
typedef __mutex __mutex_type;
private:
__mutex_type& _M_device;
__scoped_lock(const __scoped_lock&);
__scoped_lock& operator=(const __scoped_lock&);
public:
explicit __scoped_lock(__mutex_type& __name) : _M_device(__name)
{ _M_device.lock(); }
~__scoped_lock() throw()
{ _M_device.unlock(); }
};
class __cond
{
private:
__gthread_cond_t _M_cond = { { {0}, {0}, {0, 0}, {0, 0}, 0, 0, {0, 0} } };
__cond(const __cond&);
__cond& operator=(const __cond&);
public:
__cond()
{
}
void broadcast()
{
if (__gthread_active_p())
{
if (__gthread_cond_broadcast(&_M_cond) != 0)
__throw_concurrence_broadcast_error();
}
}
void wait(__mutex *mutex)
{
{
if (__gthread_cond_wait(&_M_cond, mutex->gthread_mutex()) != 0)
__throw_concurrence_wait_error();
}
}
void wait_recursive(__recursive_mutex *mutex)
{
{
if (__gthread_cond_wait_recursive(&_M_cond,
mutex->gthread_recursive_mutex())
!= 0)
__throw_concurrence_wait_error();
}
}
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename> class auto_ptr;
#pragma GCC diagnostic pop
class bad_weak_ptr : public std::exception
{
public:
virtual char const* what() const noexcept;
virtual ~bad_weak_ptr() noexcept;
};
inline void
__throw_bad_weak_ptr()
{ (throw (bad_weak_ptr())); }
using __gnu_cxx::_Lock_policy;
using __gnu_cxx::__default_lock_policy;
using __gnu_cxx::_S_single;
using __gnu_cxx::_S_mutex;
using __gnu_cxx::_S_atomic;
template<_Lock_policy _Lp>
class _Mutex_base
{
protected:
enum { _S_need_barriers = 0 };
};
template<>
class _Mutex_base<_S_mutex>
: public __gnu_cxx::__mutex
{
protected:
enum { _S_need_barriers = 1 };
};
template<_Lock_policy _Lp = __default_lock_policy>
class _Sp_counted_base
: public _Mutex_base<_Lp>
{
public:
_Sp_counted_base() noexcept
: _M_use_count(1), _M_weak_count(1) { }
virtual
~_Sp_counted_base() noexcept
{ }
virtual void
_M_dispose() noexcept = 0;
virtual void
_M_destroy() noexcept
{ delete this; }
virtual void*
_M_get_deleter(const std::type_info&) noexcept = 0;
void
_M_add_ref_copy()
{ __gnu_cxx::__atomic_add_dispatch(&_M_use_count, 1); }
void
_M_add_ref_lock()
{
if (!_M_add_ref_lock_nothrow())
__throw_bad_weak_ptr();
}
bool
_M_add_ref_lock_nothrow() noexcept;
void
_M_release() noexcept;
void
_M_release_last_use() noexcept
{
;
_M_dispose();
if (_Mutex_base<_Lp>::_S_need_barriers)
{
__atomic_thread_fence (4);
}
;
if (__gnu_cxx::__exchange_and_add_dispatch(&_M_weak_count,
-1) == 1)
{
;
_M_destroy();
}
}
__attribute__((__noinline__))
void
_M_release_last_use_cold() noexcept
{ _M_release_last_use(); }
void
_M_weak_add_ref() noexcept
{ __gnu_cxx::__atomic_add_dispatch(&_M_weak_count, 1); }
void
_M_weak_release() noexcept
{
;
if (__gnu_cxx::__exchange_and_add_dispatch(&_M_weak_count, -1) == 1)
{
;
if (_Mutex_base<_Lp>::_S_need_barriers)
{
__atomic_thread_fence (4);
}
_M_destroy();
}
}
long
_M_get_use_count() const noexcept
{
return __atomic_load_n(&_M_use_count, 0);
}
private:
_Sp_counted_base(_Sp_counted_base const&) = delete;
_Sp_counted_base& operator=(_Sp_counted_base const&) = delete;
_Atomic_word _M_use_count;
_Atomic_word _M_weak_count;
};
template<>
inline bool
_Sp_counted_base<_S_single>::
_M_add_ref_lock_nothrow() noexcept
{
if (_M_use_count == 0)
return false;
++_M_use_count;
return true;
}
template<>
inline bool
_Sp_counted_base<_S_mutex>::
_M_add_ref_lock_nothrow() noexcept
{
__gnu_cxx::__scoped_lock sentry(*this);
if (__gnu_cxx::__exchange_and_add_dispatch(&_M_use_count, 1) == 0)
{
_M_use_count = 0;
return false;
}
return true;
}
template<>
inline bool
_Sp_counted_base<_S_atomic>::
_M_add_ref_lock_nothrow() noexcept
{
_Atomic_word __count = _M_get_use_count();
do
{
if (__count == 0)
return false;
}
while (!__atomic_compare_exchange_n(&_M_use_count, &__count, __count + 1,
true, 4,
0));
return true;
}
template<>
inline void
_Sp_counted_base<_S_single>::_M_add_ref_copy()
{ ++_M_use_count; }
template<>
inline void
_Sp_counted_base<_S_single>::_M_release() noexcept
{
if (--_M_use_count == 0)
{
_M_dispose();
if (--_M_weak_count == 0)
_M_destroy();
}
}
template<>
inline void
_Sp_counted_base<_S_mutex>::_M_release() noexcept
{
;
if (__gnu_cxx::__exchange_and_add_dispatch(&_M_use_count, -1) == 1)
{
_M_release_last_use();
}
}
template<>
inline void
_Sp_counted_base<_S_atomic>::_M_release() noexcept
{
;
constexpr bool __lock_free
= __atomic_always_lock_free(sizeof(long long), 0)
&& __atomic_always_lock_free(sizeof(_Atomic_word), 0);
constexpr bool __double_word
= sizeof(long long) == 2 * sizeof(_Atomic_word);
constexpr bool __aligned = __alignof(long long) <= alignof(void*);
if constexpr (__lock_free && __double_word && __aligned)
{
constexpr int __wordbits = 8 * sizeof(_Atomic_word);
constexpr int __shiftbits = __double_word ? __wordbits : 0;
constexpr long long __unique_ref = 1LL + (1LL << __shiftbits);
auto __both_counts = reinterpret_cast<long long*>(&_M_use_count);
;
if (__atomic_load_n(__both_counts, 2) == __unique_ref)
{
_M_weak_count = _M_use_count = 0;
;
;
_M_dispose();
_M_destroy();
return;
}
if (__gnu_cxx::__exchange_and_add_dispatch(&_M_use_count, -1) == 1)
[[__unlikely__]]
{
_M_release_last_use_cold();
return;
}
}
else
if (__gnu_cxx::__exchange_and_add_dispatch(&_M_use_count, -1) == 1)
{
_M_release_last_use();
}
}
template<>
inline void
_Sp_counted_base<_S_single>::_M_weak_add_ref() noexcept
{ ++_M_weak_count; }
template<>
inline void
_Sp_counted_base<_S_single>::_M_weak_release() noexcept
{
if (--_M_weak_count == 0)
_M_destroy();
}
template<>
inline long
_Sp_counted_base<_S_single>::_M_get_use_count() const noexcept
{ return _M_use_count; }
template<typename _Tp, _Lock_policy _Lp = __default_lock_policy>
class __shared_ptr;
template<typename _Tp, _Lock_policy _Lp = __default_lock_policy>
class __weak_ptr;
template<typename _Tp, _Lock_policy _Lp = __default_lock_policy>
class __enable_shared_from_this;
template<typename _Tp>
class shared_ptr;
template<typename _Tp>
class weak_ptr;
template<typename _Tp>
struct owner_less;
template<typename _Tp>
class enable_shared_from_this;
template<_Lock_policy _Lp = __default_lock_policy>
class __weak_count;
template<_Lock_policy _Lp = __default_lock_policy>
class __shared_count;
template<typename _Ptr, _Lock_policy _Lp>
class _Sp_counted_ptr final : public _Sp_counted_base<_Lp>
{
public:
explicit
_Sp_counted_ptr(_Ptr __p) noexcept
: _M_ptr(__p) { }
virtual void
_M_dispose() noexcept
{ delete _M_ptr; }
virtual void
_M_destroy() noexcept
{ delete this; }
virtual void*
_M_get_deleter(const std::type_info&) noexcept
{ return nullptr; }
_Sp_counted_ptr(const _Sp_counted_ptr&) = delete;
_Sp_counted_ptr& operator=(const _Sp_counted_ptr&) = delete;
private:
_Ptr _M_ptr;
};
template<>
inline void
_Sp_counted_ptr<nullptr_t, _S_single>::_M_dispose() noexcept { }
template<>
inline void
_Sp_counted_ptr<nullptr_t, _S_mutex>::_M_dispose() noexcept { }
template<>
inline void
_Sp_counted_ptr<nullptr_t, _S_atomic>::_M_dispose() noexcept { }
template<int _Nm, typename _Tp,
bool __use_ebo = !__is_final(_Tp) && __is_empty(_Tp)>
struct _Sp_ebo_helper;
template<int _Nm, typename _Tp>
struct _Sp_ebo_helper<_Nm, _Tp, true> : private _Tp
{
explicit _Sp_ebo_helper(const _Tp& __tp) : _Tp(__tp) { }
explicit _Sp_ebo_helper(_Tp&& __tp) : _Tp(std::move(__tp)) { }
static _Tp&
_S_get(_Sp_ebo_helper& __eboh) { return static_cast<_Tp&>(__eboh); }
};
template<int _Nm, typename _Tp>
struct _Sp_ebo_helper<_Nm, _Tp, false>
{
explicit _Sp_ebo_helper(const _Tp& __tp) : _M_tp(__tp) { }
explicit _Sp_ebo_helper(_Tp&& __tp) : _M_tp(std::move(__tp)) { }
static _Tp&
_S_get(_Sp_ebo_helper& __eboh)
{ return __eboh._M_tp; }
private:
_Tp _M_tp;
};
template<typename _Ptr, typename _Deleter, typename _Alloc, _Lock_policy _Lp>
class _Sp_counted_deleter final : public _Sp_counted_base<_Lp>
{
class _Impl : _Sp_ebo_helper<0, _Deleter>, _Sp_ebo_helper<1, _Alloc>
{
typedef _Sp_ebo_helper<0, _Deleter> _Del_base;
typedef _Sp_ebo_helper<1, _Alloc> _Alloc_base;
public:
_Impl(_Ptr __p, _Deleter __d, const _Alloc& __a) noexcept
: _Del_base(std::move(__d)), _Alloc_base(__a), _M_ptr(__p)
{ }
_Deleter& _M_del() noexcept { return _Del_base::_S_get(*this); }
_Alloc& _M_alloc() noexcept { return _Alloc_base::_S_get(*this); }
_Ptr _M_ptr;
};
public:
using __allocator_type = __alloc_rebind<_Alloc, _Sp_counted_deleter>;
_Sp_counted_deleter(_Ptr __p, _Deleter __d) noexcept
: _M_impl(__p, std::move(__d), _Alloc()) { }
_Sp_counted_deleter(_Ptr __p, _Deleter __d, const _Alloc& __a) noexcept
: _M_impl(__p, std::move(__d), __a) { }
~_Sp_counted_deleter() noexcept { }
virtual void
_M_dispose() noexcept
{ _M_impl._M_del()(_M_impl._M_ptr); }
virtual void
_M_destroy() noexcept
{
__allocator_type __a(_M_impl._M_alloc());
__allocated_ptr<__allocator_type> __guard_ptr{ __a, this };
this->~_Sp_counted_deleter();
}
virtual void*
_M_get_deleter(const type_info& __ti [[__gnu__::__unused__]]) noexcept
{
return __ti == typeid(_Deleter)
? std::__addressof(_M_impl._M_del())
: nullptr;
}
private:
_Impl _M_impl;
};
struct _Sp_make_shared_tag
{
private:
template<typename _Tp, typename _Alloc, _Lock_policy _Lp>
friend class _Sp_counted_ptr_inplace;
static const type_info&
_S_ti() noexcept __attribute__ ((__visibility__ ("default")))
{
alignas(type_info) static constexpr char __tag[sizeof(type_info)] = { };
return reinterpret_cast<const type_info&>(__tag);
}
static bool _S_eq(const type_info&) noexcept;
};
template<typename _Alloc>
struct _Sp_alloc_shared_tag
{
const _Alloc& _M_a;
};
template<typename _Tp, typename _Alloc, _Lock_policy _Lp>
class _Sp_counted_ptr_inplace final : public _Sp_counted_base<_Lp>
{
class _Impl : _Sp_ebo_helper<0, _Alloc>
{
typedef _Sp_ebo_helper<0, _Alloc> _A_base;
public:
explicit _Impl(_Alloc __a) noexcept : _A_base(__a) { }
_Alloc& _M_alloc() noexcept { return _A_base::_S_get(*this); }
__gnu_cxx::__aligned_buffer<_Tp> _M_storage;
};
public:
using __allocator_type = __alloc_rebind<_Alloc, _Sp_counted_ptr_inplace>;
template<typename... _Args>
_Sp_counted_ptr_inplace(_Alloc __a, _Args&&... __args)
: _M_impl(__a)
{
allocator_traits<_Alloc>::construct(__a, _M_ptr(),
std::forward<_Args>(__args)...);
}
~_Sp_counted_ptr_inplace() noexcept { }
virtual void
_M_dispose() noexcept
{
allocator_traits<_Alloc>::destroy(_M_impl._M_alloc(), _M_ptr());
}
virtual void
_M_destroy() noexcept
{
__allocator_type __a(_M_impl._M_alloc());
__allocated_ptr<__allocator_type> __guard_ptr{ __a, this };
this->~_Sp_counted_ptr_inplace();
}
private:
friend class __shared_count<_Lp>;
virtual void*
_M_get_deleter(const std::type_info& __ti) noexcept override
{
auto __ptr = const_cast<typename remove_cv<_Tp>::type*>(_M_ptr());
if (&__ti == &_Sp_make_shared_tag::_S_ti()
||
__ti == typeid(_Sp_make_shared_tag)
)
return __ptr;
return nullptr;
}
_Tp* _M_ptr() noexcept { return _M_impl._M_storage._M_ptr(); }
_Impl _M_impl;
};
struct __sp_array_delete
{
template<typename _Yp>
void operator()(_Yp* __p) const { delete[] __p; }
};
template<_Lock_policy _Lp>
class __shared_count
{
template<typename _Tp>
struct __not_alloc_shared_tag { using type = void; };
template<typename _Tp>
struct __not_alloc_shared_tag<_Sp_alloc_shared_tag<_Tp>> { };
public:
constexpr __shared_count() noexcept : _M_pi(0)
{ }
template<typename _Ptr>
explicit
__shared_count(_Ptr __p) : _M_pi(0)
{
try
{
_M_pi = new _Sp_counted_ptr<_Ptr, _Lp>(__p);
}
catch(...)
{
delete __p;
throw;
}
}
template<typename _Ptr>
__shared_count(_Ptr __p, false_type)
: __shared_count(__p)
{ }
template<typename _Ptr>
__shared_count(_Ptr __p, true_type)
: __shared_count(__p, __sp_array_delete{}, allocator<void>())
{ }
template<typename _Ptr, typename _Deleter,
typename = typename __not_alloc_shared_tag<_Deleter>::type>
__shared_count(_Ptr __p, _Deleter __d)
: __shared_count(__p, std::move(__d), allocator<void>())
{ }
template<typename _Ptr, typename _Deleter, typename _Alloc,
typename = typename __not_alloc_shared_tag<_Deleter>::type>
__shared_count(_Ptr __p, _Deleter __d, _Alloc __a) : _M_pi(0)
{
typedef _Sp_counted_deleter<_Ptr, _Deleter, _Alloc, _Lp> _Sp_cd_type;
try
{
typename _Sp_cd_type::__allocator_type __a2(__a);
auto __guard = std::__allocate_guarded(__a2);
_Sp_cd_type* __mem = __guard.get();
::new (__mem) _Sp_cd_type(__p, std::move(__d), std::move(__a));
_M_pi = __mem;
__guard = nullptr;
}
catch(...)
{
__d(__p);
throw;
}
}
template<typename _Tp, typename _Alloc, typename... _Args>
__shared_count(_Tp*& __p, _Sp_alloc_shared_tag<_Alloc> __a,
_Args&&... __args)
{
typedef _Sp_counted_ptr_inplace<_Tp, _Alloc, _Lp> _Sp_cp_type;
typename _Sp_cp_type::__allocator_type __a2(__a._M_a);
auto __guard = std::__allocate_guarded(__a2);
_Sp_cp_type* __mem = __guard.get();
auto __pi = ::new (__mem)
_Sp_cp_type(__a._M_a, std::forward<_Args>(__args)...);
__guard = nullptr;
_M_pi = __pi;
__p = __pi->_M_ptr();
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Tp>
explicit
__shared_count(std::auto_ptr<_Tp>&& __r);
#pragma GCC diagnostic pop
template<typename _Tp, typename _Del>
explicit
__shared_count(std::unique_ptr<_Tp, _Del>&& __r) : _M_pi(0)
{
if (__r.get() == nullptr)
return;
using _Ptr = typename unique_ptr<_Tp, _Del>::pointer;
using _Del2 = __conditional_t<is_reference<_Del>::value,
reference_wrapper<typename remove_reference<_Del>::type>,
_Del>;
using _Sp_cd_type
= _Sp_counted_deleter<_Ptr, _Del2, allocator<void>, _Lp>;
using _Alloc = allocator<_Sp_cd_type>;
using _Alloc_traits = allocator_traits<_Alloc>;
_Alloc __a;
_Sp_cd_type* __mem = _Alloc_traits::allocate(__a, 1);
_Alloc_traits::construct(__a, __mem, __r.release(),
std::forward<_Del>(__r.get_deleter()));
_M_pi = __mem;
}
explicit __shared_count(const __weak_count<_Lp>& __r);
explicit
__shared_count(const __weak_count<_Lp>& __r, std::nothrow_t) noexcept;
~__shared_count() noexcept
{
if (_M_pi != nullptr)
_M_pi->_M_release();
}
__shared_count(const __shared_count& __r) noexcept
: _M_pi(__r._M_pi)
{
if (_M_pi != nullptr)
_M_pi->_M_add_ref_copy();
}
__shared_count&
operator=(const __shared_count& __r) noexcept
{
_Sp_counted_base<_Lp>* __tmp = __r._M_pi;
if (__tmp != _M_pi)
{
if (__tmp != nullptr)
__tmp->_M_add_ref_copy();
if (_M_pi != nullptr)
_M_pi->_M_release();
_M_pi = __tmp;
}
return *this;
}
void
_M_swap(__shared_count& __r) noexcept
{
_Sp_counted_base<_Lp>* __tmp = __r._M_pi;
__r._M_pi = _M_pi;
_M_pi = __tmp;
}
long
_M_get_use_count() const noexcept
{ return _M_pi ? _M_pi->_M_get_use_count() : 0; }
bool
_M_unique() const noexcept
{ return this->_M_get_use_count() == 1; }
void*
_M_get_deleter(const std::type_info& __ti) const noexcept
{ return _M_pi ? _M_pi->_M_get_deleter(__ti) : nullptr; }
bool
_M_less(const __shared_count& __rhs) const noexcept
{ return std::less<_Sp_counted_base<_Lp>*>()(this->_M_pi, __rhs._M_pi); }
bool
_M_less(const __weak_count<_Lp>& __rhs) const noexcept
{ return std::less<_Sp_counted_base<_Lp>*>()(this->_M_pi, __rhs._M_pi); }
friend inline bool
operator==(const __shared_count& __a, const __shared_count& __b) noexcept
{ return __a._M_pi == __b._M_pi; }
private:
friend class __weak_count<_Lp>;
_Sp_counted_base<_Lp>* _M_pi;
};
template<_Lock_policy _Lp>
class __weak_count
{
public:
constexpr __weak_count() noexcept : _M_pi(nullptr)
{ }
__weak_count(const __shared_count<_Lp>& __r) noexcept
: _M_pi(__r._M_pi)
{
if (_M_pi != nullptr)
_M_pi->_M_weak_add_ref();
}
__weak_count(const __weak_count& __r) noexcept
: _M_pi(__r._M_pi)
{
if (_M_pi != nullptr)
_M_pi->_M_weak_add_ref();
}
__weak_count(__weak_count&& __r) noexcept
: _M_pi(__r._M_pi)
{ __r._M_pi = nullptr; }
~__weak_count() noexcept
{
if (_M_pi != nullptr)
_M_pi->_M_weak_release();
}
__weak_count&
operator=(const __shared_count<_Lp>& __r) noexcept
{
_Sp_counted_base<_Lp>* __tmp = __r._M_pi;
if (__tmp != nullptr)
__tmp->_M_weak_add_ref();
if (_M_pi != nullptr)
_M_pi->_M_weak_release();
_M_pi = __tmp;
return *this;
}
__weak_count&
operator=(const __weak_count& __r) noexcept
{
_Sp_counted_base<_Lp>* __tmp = __r._M_pi;
if (__tmp != nullptr)
__tmp->_M_weak_add_ref();
if (_M_pi != nullptr)
_M_pi->_M_weak_release();
_M_pi = __tmp;
return *this;
}
__weak_count&
operator=(__weak_count&& __r) noexcept
{
if (_M_pi != nullptr)
_M_pi->_M_weak_release();
_M_pi = __r._M_pi;
__r._M_pi = nullptr;
return *this;
}
void
_M_swap(__weak_count& __r) noexcept
{
_Sp_counted_base<_Lp>* __tmp = __r._M_pi;
__r._M_pi = _M_pi;
_M_pi = __tmp;
}
long
_M_get_use_count() const noexcept
{ return _M_pi != nullptr ? _M_pi->_M_get_use_count() : 0; }
bool
_M_less(const __weak_count& __rhs) const noexcept
{ return std::less<_Sp_counted_base<_Lp>*>()(this->_M_pi, __rhs._M_pi); }
bool
_M_less(const __shared_count<_Lp>& __rhs) const noexcept
{ return std::less<_Sp_counted_base<_Lp>*>()(this->_M_pi, __rhs._M_pi); }
friend inline bool
operator==(const __weak_count& __a, const __weak_count& __b) noexcept
{ return __a._M_pi == __b._M_pi; }
private:
friend class __shared_count<_Lp>;
_Sp_counted_base<_Lp>* _M_pi;
};
template<_Lock_policy _Lp>
inline
__shared_count<_Lp>::__shared_count(const __weak_count<_Lp>& __r)
: _M_pi(__r._M_pi)
{
if (_M_pi == nullptr || !_M_pi->_M_add_ref_lock_nothrow())
__throw_bad_weak_ptr();
}
template<_Lock_policy _Lp>
inline
__shared_count<_Lp>::
__shared_count(const __weak_count<_Lp>& __r, std::nothrow_t) noexcept
: _M_pi(__r._M_pi)
{
if (_M_pi && !_M_pi->_M_add_ref_lock_nothrow())
_M_pi = nullptr;
}
template<typename _Yp_ptr, typename _Tp_ptr>
struct __sp_compatible_with
: false_type
{ };
template<typename _Yp, typename _Tp>
struct __sp_compatible_with<_Yp*, _Tp*>
: is_convertible<_Yp*, _Tp*>::type
{ };
template<typename _Up, size_t _Nm>
struct __sp_compatible_with<_Up(*)[_Nm], _Up(*)[]>
: true_type
{ };
template<typename _Up, size_t _Nm>
struct __sp_compatible_with<_Up(*)[_Nm], const _Up(*)[]>
: true_type
{ };
template<typename _Up, size_t _Nm>
struct __sp_compatible_with<_Up(*)[_Nm], volatile _Up(*)[]>
: true_type
{ };
template<typename _Up, size_t _Nm>
struct __sp_compatible_with<_Up(*)[_Nm], const volatile _Up(*)[]>
: true_type
{ };
template<typename _Up, size_t _Nm, typename _Yp, typename = void>
struct __sp_is_constructible_arrN
: false_type
{ };
template<typename _Up, size_t _Nm, typename _Yp>
struct __sp_is_constructible_arrN<_Up, _Nm, _Yp, __void_t<_Yp[_Nm]>>
: is_convertible<_Yp(*)[_Nm], _Up(*)[_Nm]>::type
{ };
template<typename _Up, typename _Yp, typename = void>
struct __sp_is_constructible_arr
: false_type
{ };
template<typename _Up, typename _Yp>
struct __sp_is_constructible_arr<_Up, _Yp, __void_t<_Yp[]>>
: is_convertible<_Yp(*)[], _Up(*)[]>::type
{ };
template<typename _Tp, typename _Yp>
struct __sp_is_constructible;
template<typename _Up, size_t _Nm, typename _Yp>
struct __sp_is_constructible<_Up[_Nm], _Yp>
: __sp_is_constructible_arrN<_Up, _Nm, _Yp>::type
{ };
template<typename _Up, typename _Yp>
struct __sp_is_constructible<_Up[], _Yp>
: __sp_is_constructible_arr<_Up, _Yp>::type
{ };
template<typename _Tp, typename _Yp>
struct __sp_is_constructible
: is_convertible<_Yp*, _Tp*>::type
{ };
template<typename _Tp, _Lock_policy _Lp,
bool = is_array<_Tp>::value, bool = is_void<_Tp>::value>
class __shared_ptr_access
{
public:
using element_type = _Tp;
element_type&
operator*() const noexcept
{
do { if (std::__is_constant_evaluated() && !bool(_M_get() != nullptr)) __builtin_unreachable(); } while (false);
return *_M_get();
}
element_type*
operator->() const noexcept
{
;
return _M_get();
}
private:
element_type*
_M_get() const noexcept
{ return static_cast<const __shared_ptr<_Tp, _Lp>*>(this)->get(); }
};
template<typename _Tp, _Lock_policy _Lp>
class __shared_ptr_access<_Tp, _Lp, false, true>
{
public:
using element_type = _Tp;
element_type*
operator->() const noexcept
{
auto __ptr = static_cast<const __shared_ptr<_Tp, _Lp>*>(this)->get();
;
return __ptr;
}
};
template<typename _Tp, _Lock_policy _Lp>
class __shared_ptr_access<_Tp, _Lp, true, false>
{
public:
using element_type = typename remove_extent<_Tp>::type;
element_type&
operator[](ptrdiff_t __i) const noexcept
{
do { if (std::__is_constant_evaluated() && !bool(_M_get() != nullptr)) __builtin_unreachable(); } while (false);
do { if (std::__is_constant_evaluated() && !bool(!extent<_Tp>::value || __i < extent<_Tp>::value)) __builtin_unreachable(); } while (false);
return _M_get()[__i];
}
private:
element_type*
_M_get() const noexcept
{ return static_cast<const __shared_ptr<_Tp, _Lp>*>(this)->get(); }
};
template<typename _Tp, _Lock_policy _Lp>
class __shared_ptr
: public __shared_ptr_access<_Tp, _Lp>
{
public:
using element_type = typename remove_extent<_Tp>::type;
private:
template<typename _Yp>
using _SafeConv
= typename enable_if<__sp_is_constructible<_Tp, _Yp>::value>::type;
template<typename _Yp, typename _Res = void>
using _Compatible = typename
enable_if<__sp_compatible_with<_Yp*, _Tp*>::value, _Res>::type;
template<typename _Yp>
using _Assignable = _Compatible<_Yp, __shared_ptr&>;
template<typename _Yp, typename _Del, typename _Res = void,
typename _Ptr = typename unique_ptr<_Yp, _Del>::pointer>
using _UniqCompatible = __enable_if_t<__and_<
__sp_compatible_with<_Yp*, _Tp*>,
is_convertible<_Ptr, element_type*>,
is_move_constructible<_Del>
>::value, _Res>;
template<typename _Yp, typename _Del>
using _UniqAssignable = _UniqCompatible<_Yp, _Del, __shared_ptr&>;
public:
using weak_type = __weak_ptr<_Tp, _Lp>;
constexpr __shared_ptr() noexcept
: _M_ptr(0), _M_refcount()
{ }
template<typename _Yp, typename = _SafeConv<_Yp>>
explicit
__shared_ptr(_Yp* __p)
: _M_ptr(__p), _M_refcount(__p, typename is_array<_Tp>::type())
{
static_assert( !is_void<_Yp>::value, "incomplete type" );
static_assert( sizeof(_Yp) > 0, "incomplete type" );
_M_enable_shared_from_this_with(__p);
}
template<typename _Yp, typename _Deleter, typename = _SafeConv<_Yp>>
__shared_ptr(_Yp* __p, _Deleter __d)
: _M_ptr(__p), _M_refcount(__p, std::move(__d))
{
static_assert(__is_invocable<_Deleter&, _Yp*&>::value,
"deleter expression d(p) is well-formed");
_M_enable_shared_from_this_with(__p);
}
template<typename _Yp, typename _Deleter, typename _Alloc,
typename = _SafeConv<_Yp>>
__shared_ptr(_Yp* __p, _Deleter __d, _Alloc __a)
: _M_ptr(__p), _M_refcount(__p, std::move(__d), std::move(__a))
{
static_assert(__is_invocable<_Deleter&, _Yp*&>::value,
"deleter expression d(p) is well-formed");
_M_enable_shared_from_this_with(__p);
}
template<typename _Deleter>
__shared_ptr(nullptr_t __p, _Deleter __d)
: _M_ptr(0), _M_refcount(__p, std::move(__d))
{ }
template<typename _Deleter, typename _Alloc>
__shared_ptr(nullptr_t __p, _Deleter __d, _Alloc __a)
: _M_ptr(0), _M_refcount(__p, std::move(__d), std::move(__a))
{ }
template<typename _Yp>
__shared_ptr(const __shared_ptr<_Yp, _Lp>& __r,
element_type* __p) noexcept
: _M_ptr(__p), _M_refcount(__r._M_refcount)
{ }
template<typename _Yp>
__shared_ptr(__shared_ptr<_Yp, _Lp>&& __r,
element_type* __p) noexcept
: _M_ptr(__p), _M_refcount()
{
_M_refcount._M_swap(__r._M_refcount);
__r._M_ptr = nullptr;
}
__shared_ptr(const __shared_ptr&) noexcept = default;
__shared_ptr& operator=(const __shared_ptr&) noexcept = default;
~__shared_ptr() = default;
template<typename _Yp, typename = _Compatible<_Yp>>
__shared_ptr(const __shared_ptr<_Yp, _Lp>& __r) noexcept
: _M_ptr(__r._M_ptr), _M_refcount(__r._M_refcount)
{ }
__shared_ptr(__shared_ptr&& __r) noexcept
: _M_ptr(__r._M_ptr), _M_refcount()
{
_M_refcount._M_swap(__r._M_refcount);
__r._M_ptr = nullptr;
}
template<typename _Yp, typename = _Compatible<_Yp>>
__shared_ptr(__shared_ptr<_Yp, _Lp>&& __r) noexcept
: _M_ptr(__r._M_ptr), _M_refcount()
{
_M_refcount._M_swap(__r._M_refcount);
__r._M_ptr = nullptr;
}
template<typename _Yp, typename = _Compatible<_Yp>>
explicit __shared_ptr(const __weak_ptr<_Yp, _Lp>& __r)
: _M_refcount(__r._M_refcount)
{
_M_ptr = __r._M_ptr;
}
template<typename _Yp, typename _Del,
typename = _UniqCompatible<_Yp, _Del>>
__shared_ptr(unique_ptr<_Yp, _Del>&& __r)
: _M_ptr(__r.get()), _M_refcount()
{
auto __raw = __to_address(__r.get());
_M_refcount = __shared_count<_Lp>(std::move(__r));
_M_enable_shared_from_this_with(__raw);
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Yp, typename = _Compatible<_Yp>>
__shared_ptr(auto_ptr<_Yp>&& __r);
#pragma GCC diagnostic pop
constexpr __shared_ptr(nullptr_t) noexcept : __shared_ptr() { }
template<typename _Yp>
_Assignable<_Yp>
operator=(const __shared_ptr<_Yp, _Lp>& __r) noexcept
{
_M_ptr = __r._M_ptr;
_M_refcount = __r._M_refcount;
return *this;
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Yp>
_Assignable<_Yp>
operator=(auto_ptr<_Yp>&& __r)
{
__shared_ptr(std::move(__r)).swap(*this);
return *this;
}
#pragma GCC diagnostic pop
__shared_ptr&
operator=(__shared_ptr&& __r) noexcept
{
__shared_ptr(std::move(__r)).swap(*this);
return *this;
}
template<class _Yp>
_Assignable<_Yp>
operator=(__shared_ptr<_Yp, _Lp>&& __r) noexcept
{
__shared_ptr(std::move(__r)).swap(*this);
return *this;
}
template<typename _Yp, typename _Del>
_UniqAssignable<_Yp, _Del>
operator=(unique_ptr<_Yp, _Del>&& __r)
{
__shared_ptr(std::move(__r)).swap(*this);
return *this;
}
void
reset() noexcept
{ __shared_ptr().swap(*this); }
template<typename _Yp>
_SafeConv<_Yp>
reset(_Yp* __p)
{
do { if (std::__is_constant_evaluated() && !bool(__p == nullptr || __p != _M_ptr)) __builtin_unreachable(); } while (false);
__shared_ptr(__p).swap(*this);
}
template<typename _Yp, typename _Deleter>
_SafeConv<_Yp>
reset(_Yp* __p, _Deleter __d)
{ __shared_ptr(__p, std::move(__d)).swap(*this); }
template<typename _Yp, typename _Deleter, typename _Alloc>
_SafeConv<_Yp>
reset(_Yp* __p, _Deleter __d, _Alloc __a)
{ __shared_ptr(__p, std::move(__d), std::move(__a)).swap(*this); }
element_type*
get() const noexcept
{ return _M_ptr; }
explicit operator bool() const noexcept
{ return _M_ptr != nullptr; }
bool
unique() const noexcept
{ return _M_refcount._M_unique(); }
long
use_count() const noexcept
{ return _M_refcount._M_get_use_count(); }
void
swap(__shared_ptr<_Tp, _Lp>& __other) noexcept
{
std::swap(_M_ptr, __other._M_ptr);
_M_refcount._M_swap(__other._M_refcount);
}
template<typename _Tp1>
bool
owner_before(__shared_ptr<_Tp1, _Lp> const& __rhs) const noexcept
{ return _M_refcount._M_less(__rhs._M_refcount); }
template<typename _Tp1>
bool
owner_before(__weak_ptr<_Tp1, _Lp> const& __rhs) const noexcept
{ return _M_refcount._M_less(__rhs._M_refcount); }
protected:
template<typename _Alloc, typename... _Args>
__shared_ptr(_Sp_alloc_shared_tag<_Alloc> __tag, _Args&&... __args)
: _M_ptr(), _M_refcount(_M_ptr, __tag, std::forward<_Args>(__args)...)
{ _M_enable_shared_from_this_with(_M_ptr); }
template<typename _Tp1, _Lock_policy _Lp1, typename _Alloc,
typename... _Args>
friend __shared_ptr<_Tp1, _Lp1>
__allocate_shared(const _Alloc& __a, _Args&&... __args);
__shared_ptr(const __weak_ptr<_Tp, _Lp>& __r, std::nothrow_t) noexcept
: _M_refcount(__r._M_refcount, std::nothrow)
{
_M_ptr = _M_refcount._M_get_use_count() ? __r._M_ptr : nullptr;
}
friend class __weak_ptr<_Tp, _Lp>;
private:
template<typename _Yp>
using __esft_base_t = decltype(__enable_shared_from_this_base(
std::declval<const __shared_count<_Lp>&>(),
std::declval<_Yp*>()));
template<typename _Yp, typename = void>
struct __has_esft_base
: false_type { };
template<typename _Yp>
struct __has_esft_base<_Yp, __void_t<__esft_base_t<_Yp>>>
: __not_<is_array<_Tp>> { };
template<typename _Yp, typename _Yp2 = typename remove_cv<_Yp>::type>
typename enable_if<__has_esft_base<_Yp2>::value>::type
_M_enable_shared_from_this_with(_Yp* __p) noexcept
{
if (auto __base = __enable_shared_from_this_base(_M_refcount, __p))
__base->_M_weak_assign(const_cast<_Yp2*>(__p), _M_refcount);
}
template<typename _Yp, typename _Yp2 = typename remove_cv<_Yp>::type>
typename enable_if<!__has_esft_base<_Yp2>::value>::type
_M_enable_shared_from_this_with(_Yp*) noexcept
{ }
void*
_M_get_deleter(const std::type_info& __ti) const noexcept
{ return _M_refcount._M_get_deleter(__ti); }
template<typename _Tp1, _Lock_policy _Lp1> friend class __shared_ptr;
template<typename _Tp1, _Lock_policy _Lp1> friend class __weak_ptr;
template<typename _Del, typename _Tp1, _Lock_policy _Lp1>
friend _Del* get_deleter(const __shared_ptr<_Tp1, _Lp1>&) noexcept;
template<typename _Del, typename _Tp1>
friend _Del* get_deleter(const shared_ptr<_Tp1>&) noexcept;
element_type* _M_ptr;
__shared_count<_Lp> _M_refcount;
};
template<typename _Tp1, typename _Tp2, _Lock_policy _Lp>
inline bool
operator==(const __shared_ptr<_Tp1, _Lp>& __a,
const __shared_ptr<_Tp2, _Lp>& __b) noexcept
{ return __a.get() == __b.get(); }
template<typename _Tp, _Lock_policy _Lp>
inline bool
operator==(const __shared_ptr<_Tp, _Lp>& __a, nullptr_t) noexcept
{ return !__a; }
template<typename _Tp, _Lock_policy _Lp>
inline bool
operator==(nullptr_t, const __shared_ptr<_Tp, _Lp>& __a) noexcept
{ return !__a; }
template<typename _Tp1, typename _Tp2, _Lock_policy _Lp>
inline bool
operator!=(const __shared_ptr<_Tp1, _Lp>& __a,
const __shared_ptr<_Tp2, _Lp>& __b) noexcept
{ return __a.get() != __b.get(); }
template<typename _Tp, _Lock_policy _Lp>
inline bool
operator!=(const __shared_ptr<_Tp, _Lp>& __a, nullptr_t) noexcept
{ return (bool)__a; }
template<typename _Tp, _Lock_policy _Lp>
inline bool
operator!=(nullptr_t, const __shared_ptr<_Tp, _Lp>& __a) noexcept
{ return (bool)__a; }
template<typename _Tp, typename _Up, _Lock_policy _Lp>
inline bool
operator<(const __shared_ptr<_Tp, _Lp>& __a,
const __shared_ptr<_Up, _Lp>& __b) noexcept
{
using _Tp_elt = typename __shared_ptr<_Tp, _Lp>::element_type;
using _Up_elt = typename __shared_ptr<_Up, _Lp>::element_type;
using _Vp = typename common_type<_Tp_elt*, _Up_elt*>::type;
return less<_Vp>()(__a.get(), __b.get());
}
template<typename _Tp, _Lock_policy _Lp>
inline bool
operator<(const __shared_ptr<_Tp, _Lp>& __a, nullptr_t) noexcept
{
using _Tp_elt = typename __shared_ptr<_Tp, _Lp>::element_type;
return less<_Tp_elt*>()(__a.get(), nullptr);
}
template<typename _Tp, _Lock_policy _Lp>
inline bool
operator<(nullptr_t, const __shared_ptr<_Tp, _Lp>& __a) noexcept
{
using _Tp_elt = typename __shared_ptr<_Tp, _Lp>::element_type;
return less<_Tp_elt*>()(nullptr, __a.get());
}
template<typename _Tp1, typename _Tp2, _Lock_policy _Lp>
inline bool
operator<=(const __shared_ptr<_Tp1, _Lp>& __a,
const __shared_ptr<_Tp2, _Lp>& __b) noexcept
{ return !(__b < __a); }
template<typename _Tp, _Lock_policy _Lp>
inline bool
operator<=(const __shared_ptr<_Tp, _Lp>& __a, nullptr_t) noexcept
{ return !(nullptr < __a); }
template<typename _Tp, _Lock_policy _Lp>
inline bool
operator<=(nullptr_t, const __shared_ptr<_Tp, _Lp>& __a) noexcept
{ return !(__a < nullptr); }
template<typename _Tp1, typename _Tp2, _Lock_policy _Lp>
inline bool
operator>(const __shared_ptr<_Tp1, _Lp>& __a,
const __shared_ptr<_Tp2, _Lp>& __b) noexcept
{ return (__b < __a); }
template<typename _Tp, _Lock_policy _Lp>
inline bool
operator>(const __shared_ptr<_Tp, _Lp>& __a, nullptr_t) noexcept
{ return nullptr < __a; }
template<typename _Tp, _Lock_policy _Lp>
inline bool
operator>(nullptr_t, const __shared_ptr<_Tp, _Lp>& __a) noexcept
{ return __a < nullptr; }
template<typename _Tp1, typename _Tp2, _Lock_policy _Lp>
inline bool
operator>=(const __shared_ptr<_Tp1, _Lp>& __a,
const __shared_ptr<_Tp2, _Lp>& __b) noexcept
{ return !(__a < __b); }
template<typename _Tp, _Lock_policy _Lp>
inline bool
operator>=(const __shared_ptr<_Tp, _Lp>& __a, nullptr_t) noexcept
{ return !(__a < nullptr); }
template<typename _Tp, _Lock_policy _Lp>
inline bool
operator>=(nullptr_t, const __shared_ptr<_Tp, _Lp>& __a) noexcept
{ return !(nullptr < __a); }
template<typename _Tp, _Lock_policy _Lp>
inline void
swap(__shared_ptr<_Tp, _Lp>& __a, __shared_ptr<_Tp, _Lp>& __b) noexcept
{ __a.swap(__b); }
template<typename _Tp, typename _Tp1, _Lock_policy _Lp>
inline __shared_ptr<_Tp, _Lp>
static_pointer_cast(const __shared_ptr<_Tp1, _Lp>& __r) noexcept
{
using _Sp = __shared_ptr<_Tp, _Lp>;
return _Sp(__r, static_cast<typename _Sp::element_type*>(__r.get()));
}
template<typename _Tp, typename _Tp1, _Lock_policy _Lp>
inline __shared_ptr<_Tp, _Lp>
const_pointer_cast(const __shared_ptr<_Tp1, _Lp>& __r) noexcept
{
using _Sp = __shared_ptr<_Tp, _Lp>;
return _Sp(__r, const_cast<typename _Sp::element_type*>(__r.get()));
}
template<typename _Tp, typename _Tp1, _Lock_policy _Lp>
inline __shared_ptr<_Tp, _Lp>
dynamic_pointer_cast(const __shared_ptr<_Tp1, _Lp>& __r) noexcept
{
using _Sp = __shared_ptr<_Tp, _Lp>;
if (auto* __p = dynamic_cast<typename _Sp::element_type*>(__r.get()))
return _Sp(__r, __p);
return _Sp();
}
template<typename _Tp, typename _Tp1, _Lock_policy _Lp>
inline __shared_ptr<_Tp, _Lp>
reinterpret_pointer_cast(const __shared_ptr<_Tp1, _Lp>& __r) noexcept
{
using _Sp = __shared_ptr<_Tp, _Lp>;
return _Sp(__r, reinterpret_cast<typename _Sp::element_type*>(__r.get()));
}
template<typename _Tp, _Lock_policy _Lp>
class __weak_ptr
{
template<typename _Yp, typename _Res = void>
using _Compatible = typename
enable_if<__sp_compatible_with<_Yp*, _Tp*>::value, _Res>::type;
template<typename _Yp>
using _Assignable = _Compatible<_Yp, __weak_ptr&>;
public:
using element_type = typename remove_extent<_Tp>::type;
constexpr __weak_ptr() noexcept
: _M_ptr(nullptr), _M_refcount()
{ }
__weak_ptr(const __weak_ptr&) noexcept = default;
~__weak_ptr() = default;
template<typename _Yp, typename = _Compatible<_Yp>>
__weak_ptr(const __weak_ptr<_Yp, _Lp>& __r) noexcept
: _M_refcount(__r._M_refcount)
{ _M_ptr = __r.lock().get(); }
template<typename _Yp, typename = _Compatible<_Yp>>
__weak_ptr(const __shared_ptr<_Yp, _Lp>& __r) noexcept
: _M_ptr(__r._M_ptr), _M_refcount(__r._M_refcount)
{ }
__weak_ptr(__weak_ptr&& __r) noexcept
: _M_ptr(__r._M_ptr), _M_refcount(std::move(__r._M_refcount))
{ __r._M_ptr = nullptr; }
template<typename _Yp, typename = _Compatible<_Yp>>
__weak_ptr(__weak_ptr<_Yp, _Lp>&& __r) noexcept
: _M_ptr(__r.lock().get()), _M_refcount(std::move(__r._M_refcount))
{ __r._M_ptr = nullptr; }
__weak_ptr&
operator=(const __weak_ptr& __r) noexcept = default;
template<typename _Yp>
_Assignable<_Yp>
operator=(const __weak_ptr<_Yp, _Lp>& __r) noexcept
{
_M_ptr = __r.lock().get();
_M_refcount = __r._M_refcount;
return *this;
}
template<typename _Yp>
_Assignable<_Yp>
operator=(const __shared_ptr<_Yp, _Lp>& __r) noexcept
{
_M_ptr = __r._M_ptr;
_M_refcount = __r._M_refcount;
return *this;
}
__weak_ptr&
operator=(__weak_ptr&& __r) noexcept
{
__weak_ptr(std::move(__r)).swap(*this);
return *this;
}
template<typename _Yp>
_Assignable<_Yp>
operator=(__weak_ptr<_Yp, _Lp>&& __r) noexcept
{
_M_ptr = __r.lock().get();
_M_refcount = std::move(__r._M_refcount);
__r._M_ptr = nullptr;
return *this;
}
__shared_ptr<_Tp, _Lp>
lock() const noexcept
{ return __shared_ptr<element_type, _Lp>(*this, std::nothrow); }
long
use_count() const noexcept
{ return _M_refcount._M_get_use_count(); }
bool
expired() const noexcept
{ return _M_refcount._M_get_use_count() == 0; }
template<typename _Tp1>
bool
owner_before(const __shared_ptr<_Tp1, _Lp>& __rhs) const noexcept
{ return _M_refcount._M_less(__rhs._M_refcount); }
template<typename _Tp1>
bool
owner_before(const __weak_ptr<_Tp1, _Lp>& __rhs) const noexcept
{ return _M_refcount._M_less(__rhs._M_refcount); }
void
reset() noexcept
{ __weak_ptr().swap(*this); }
void
swap(__weak_ptr& __s) noexcept
{
std::swap(_M_ptr, __s._M_ptr);
_M_refcount._M_swap(__s._M_refcount);
}
private:
void
_M_assign(_Tp* __ptr, const __shared_count<_Lp>& __refcount) noexcept
{
if (use_count() == 0)
{
_M_ptr = __ptr;
_M_refcount = __refcount;
}
}
template<typename _Tp1, _Lock_policy _Lp1> friend class __shared_ptr;
template<typename _Tp1, _Lock_policy _Lp1> friend class __weak_ptr;
friend class __enable_shared_from_this<_Tp, _Lp>;
friend class enable_shared_from_this<_Tp>;
element_type* _M_ptr;
__weak_count<_Lp> _M_refcount;
};
template<typename _Tp, _Lock_policy _Lp>
inline void
swap(__weak_ptr<_Tp, _Lp>& __a, __weak_ptr<_Tp, _Lp>& __b) noexcept
{ __a.swap(__b); }
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Tp, typename _Tp1>
struct _Sp_owner_less : public binary_function<_Tp, _Tp, bool>
{
bool
operator()(const _Tp& __lhs, const _Tp& __rhs) const noexcept
{ return __lhs.owner_before(__rhs); }
bool
operator()(const _Tp& __lhs, const _Tp1& __rhs) const noexcept
{ return __lhs.owner_before(__rhs); }
bool
operator()(const _Tp1& __lhs, const _Tp& __rhs) const noexcept
{ return __lhs.owner_before(__rhs); }
};
#pragma GCC diagnostic pop
template<>
struct _Sp_owner_less<void, void>
{
template<typename _Tp, typename _Up>
auto
operator()(const _Tp& __lhs, const _Up& __rhs) const noexcept
-> decltype(__lhs.owner_before(__rhs))
{ return __lhs.owner_before(__rhs); }
using is_transparent = void;
};
template<typename _Tp, _Lock_policy _Lp>
struct owner_less<__shared_ptr<_Tp, _Lp>>
: public _Sp_owner_less<__shared_ptr<_Tp, _Lp>, __weak_ptr<_Tp, _Lp>>
{ };
template<typename _Tp, _Lock_policy _Lp>
struct owner_less<__weak_ptr<_Tp, _Lp>>
: public _Sp_owner_less<__weak_ptr<_Tp, _Lp>, __shared_ptr<_Tp, _Lp>>
{ };
template<typename _Tp, _Lock_policy _Lp>
class __enable_shared_from_this
{
protected:
constexpr __enable_shared_from_this() noexcept { }
__enable_shared_from_this(const __enable_shared_from_this&) noexcept { }
__enable_shared_from_this&
operator=(const __enable_shared_from_this&) noexcept
{ return *this; }
~__enable_shared_from_this() { }
public:
__shared_ptr<_Tp, _Lp>
shared_from_this()
{ return __shared_ptr<_Tp, _Lp>(this->_M_weak_this); }
__shared_ptr<const _Tp, _Lp>
shared_from_this() const
{ return __shared_ptr<const _Tp, _Lp>(this->_M_weak_this); }
__weak_ptr<_Tp, _Lp>
weak_from_this() noexcept
{ return this->_M_weak_this; }
__weak_ptr<const _Tp, _Lp>
weak_from_this() const noexcept
{ return this->_M_weak_this; }
private:
template<typename _Tp1>
void
_M_weak_assign(_Tp1* __p, const __shared_count<_Lp>& __n) const noexcept
{ _M_weak_this._M_assign(__p, __n); }
friend const __enable_shared_from_this*
__enable_shared_from_this_base(const __shared_count<_Lp>&,
const __enable_shared_from_this* __p)
{ return __p; }
template<typename, _Lock_policy>
friend class __shared_ptr;
mutable __weak_ptr<_Tp, _Lp> _M_weak_this;
};
template<typename _Tp, _Lock_policy _Lp = __default_lock_policy,
typename _Alloc, typename... _Args>
inline __shared_ptr<_Tp, _Lp>
__allocate_shared(const _Alloc& __a, _Args&&... __args)
{
static_assert(!is_array<_Tp>::value, "make_shared<T[]> not supported");
return __shared_ptr<_Tp, _Lp>(_Sp_alloc_shared_tag<_Alloc>{__a},
std::forward<_Args>(__args)...);
}
template<typename _Tp, _Lock_policy _Lp = __default_lock_policy,
typename... _Args>
inline __shared_ptr<_Tp, _Lp>
__make_shared(_Args&&... __args)
{
typedef typename std::remove_const<_Tp>::type _Tp_nc;
return std::__allocate_shared<_Tp, _Lp>(std::allocator<_Tp_nc>(),
std::forward<_Args>(__args)...);
}
template<typename _Tp, _Lock_policy _Lp>
struct hash<__shared_ptr<_Tp, _Lp>>
: public __hash_base<size_t, __shared_ptr<_Tp, _Lp>>
{
size_t
operator()(const __shared_ptr<_Tp, _Lp>& __s) const noexcept
{
return hash<typename __shared_ptr<_Tp, _Lp>::element_type*>()(
__s.get());
}
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Ch, typename _Tr, typename _Tp, _Lock_policy _Lp>
inline std::basic_ostream<_Ch, _Tr>&
operator<<(std::basic_ostream<_Ch, _Tr>& __os,
const __shared_ptr<_Tp, _Lp>& __p)
{
__os << __p.get();
return __os;
}
template<typename _Del, typename _Tp, _Lock_policy _Lp>
inline _Del*
get_deleter(const __shared_ptr<_Tp, _Lp>& __p) noexcept
{
return static_cast<_Del*>(__p._M_get_deleter(typeid(_Del)));
}
template<typename _Del, typename _Tp>
inline _Del*
get_deleter(const shared_ptr<_Tp>& __p) noexcept
{
return static_cast<_Del*>(__p._M_get_deleter(typeid(_Del)));
}
template<typename _Tp>
using _NonArray = __enable_if_t<!is_array<_Tp>::value, _Tp>;
template<typename _Tp>
class shared_ptr : public __shared_ptr<_Tp>
{
template<typename... _Args>
using _Constructible = typename enable_if<
is_constructible<__shared_ptr<_Tp>, _Args...>::value
>::type;
template<typename _Arg>
using _Assignable = typename enable_if<
is_assignable<__shared_ptr<_Tp>&, _Arg>::value, shared_ptr&
>::type;
public:
using element_type = typename __shared_ptr<_Tp>::element_type;
using weak_type = weak_ptr<_Tp>;
constexpr shared_ptr() noexcept : __shared_ptr<_Tp>() { }
shared_ptr(const shared_ptr&) noexcept = default;
template<typename _Yp, typename = _Constructible<_Yp*>>
explicit
shared_ptr(_Yp* __p) : __shared_ptr<_Tp>(__p) { }
template<typename _Yp, typename _Deleter,
typename = _Constructible<_Yp*, _Deleter>>
shared_ptr(_Yp* __p, _Deleter __d)
: __shared_ptr<_Tp>(__p, std::move(__d)) { }
template<typename _Deleter>
shared_ptr(nullptr_t __p, _Deleter __d)
: __shared_ptr<_Tp>(__p, std::move(__d)) { }
template<typename _Yp, typename _Deleter, typename _Alloc,
typename = _Constructible<_Yp*, _Deleter, _Alloc>>
shared_ptr(_Yp* __p, _Deleter __d, _Alloc __a)
: __shared_ptr<_Tp>(__p, std::move(__d), std::move(__a)) { }
template<typename _Deleter, typename _Alloc>
shared_ptr(nullptr_t __p, _Deleter __d, _Alloc __a)
: __shared_ptr<_Tp>(__p, std::move(__d), std::move(__a)) { }
template<typename _Yp>
shared_ptr(const shared_ptr<_Yp>& __r, element_type* __p) noexcept
: __shared_ptr<_Tp>(__r, __p) { }
template<typename _Yp,
typename = _Constructible<const shared_ptr<_Yp>&>>
shared_ptr(const shared_ptr<_Yp>& __r) noexcept
: __shared_ptr<_Tp>(__r) { }
shared_ptr(shared_ptr&& __r) noexcept
: __shared_ptr<_Tp>(std::move(__r)) { }
template<typename _Yp, typename = _Constructible<shared_ptr<_Yp>>>
shared_ptr(shared_ptr<_Yp>&& __r) noexcept
: __shared_ptr<_Tp>(std::move(__r)) { }
template<typename _Yp, typename = _Constructible<const weak_ptr<_Yp>&>>
explicit shared_ptr(const weak_ptr<_Yp>& __r)
: __shared_ptr<_Tp>(__r) { }
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Yp, typename = _Constructible<auto_ptr<_Yp>>>
shared_ptr(auto_ptr<_Yp>&& __r);
#pragma GCC diagnostic pop
template<typename _Yp, typename _Del,
typename = _Constructible<unique_ptr<_Yp, _Del>>>
shared_ptr(unique_ptr<_Yp, _Del>&& __r)
: __shared_ptr<_Tp>(std::move(__r)) { }
constexpr shared_ptr(nullptr_t) noexcept : shared_ptr() { }
shared_ptr& operator=(const shared_ptr&) noexcept = default;
template<typename _Yp>
_Assignable<const shared_ptr<_Yp>&>
operator=(const shared_ptr<_Yp>& __r) noexcept
{
this->__shared_ptr<_Tp>::operator=(__r);
return *this;
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Yp>
_Assignable<auto_ptr<_Yp>>
operator=(auto_ptr<_Yp>&& __r)
{
this->__shared_ptr<_Tp>::operator=(std::move(__r));
return *this;
}
#pragma GCC diagnostic pop
shared_ptr&
operator=(shared_ptr&& __r) noexcept
{
this->__shared_ptr<_Tp>::operator=(std::move(__r));
return *this;
}
template<class _Yp>
_Assignable<shared_ptr<_Yp>>
operator=(shared_ptr<_Yp>&& __r) noexcept
{
this->__shared_ptr<_Tp>::operator=(std::move(__r));
return *this;
}
template<typename _Yp, typename _Del>
_Assignable<unique_ptr<_Yp, _Del>>
operator=(unique_ptr<_Yp, _Del>&& __r)
{
this->__shared_ptr<_Tp>::operator=(std::move(__r));
return *this;
}
private:
template<typename _Alloc, typename... _Args>
shared_ptr(_Sp_alloc_shared_tag<_Alloc> __tag, _Args&&... __args)
: __shared_ptr<_Tp>(__tag, std::forward<_Args>(__args)...)
{ }
template<typename _Yp, typename _Alloc, typename... _Args>
friend shared_ptr<_NonArray<_Yp>>
allocate_shared(const _Alloc&, _Args&&...);
template<typename _Yp, typename... _Args>
friend shared_ptr<_NonArray<_Yp>>
make_shared(_Args&&...);
shared_ptr(const weak_ptr<_Tp>& __r, std::nothrow_t) noexcept
: __shared_ptr<_Tp>(__r, std::nothrow) { }
friend class weak_ptr<_Tp>;
};
template<typename _Tp>
shared_ptr(weak_ptr<_Tp>) -> shared_ptr<_Tp>;
template<typename _Tp, typename _Del>
shared_ptr(unique_ptr<_Tp, _Del>) -> shared_ptr<_Tp>;
template<typename _Tp, typename _Up>
[[__nodiscard__]] inline bool
operator==(const shared_ptr<_Tp>& __a, const shared_ptr<_Up>& __b) noexcept
{ return __a.get() == __b.get(); }
template<typename _Tp>
[[__nodiscard__]] inline bool
operator==(const shared_ptr<_Tp>& __a, nullptr_t) noexcept
{ return !__a; }
template<typename _Tp>
[[__nodiscard__]] inline bool
operator==(nullptr_t, const shared_ptr<_Tp>& __a) noexcept
{ return !__a; }
template<typename _Tp, typename _Up>
[[__nodiscard__]] inline bool
operator!=(const shared_ptr<_Tp>& __a, const shared_ptr<_Up>& __b) noexcept
{ return __a.get() != __b.get(); }
template<typename _Tp>
[[__nodiscard__]] inline bool
operator!=(const shared_ptr<_Tp>& __a, nullptr_t) noexcept
{ return (bool)__a; }
template<typename _Tp>
[[__nodiscard__]] inline bool
operator!=(nullptr_t, const shared_ptr<_Tp>& __a) noexcept
{ return (bool)__a; }
template<typename _Tp, typename _Up>
[[__nodiscard__]] inline bool
operator<(const shared_ptr<_Tp>& __a, const shared_ptr<_Up>& __b) noexcept
{
using _Tp_elt = typename shared_ptr<_Tp>::element_type;
using _Up_elt = typename shared_ptr<_Up>::element_type;
using _Vp = typename common_type<_Tp_elt*, _Up_elt*>::type;
return less<_Vp>()(__a.get(), __b.get());
}
template<typename _Tp>
[[__nodiscard__]] inline bool
operator<(const shared_ptr<_Tp>& __a, nullptr_t) noexcept
{
using _Tp_elt = typename shared_ptr<_Tp>::element_type;
return less<_Tp_elt*>()(__a.get(), nullptr);
}
template<typename _Tp>
[[__nodiscard__]] inline bool
operator<(nullptr_t, const shared_ptr<_Tp>& __a) noexcept
{
using _Tp_elt = typename shared_ptr<_Tp>::element_type;
return less<_Tp_elt*>()(nullptr, __a.get());
}
template<typename _Tp, typename _Up>
[[__nodiscard__]] inline bool
operator<=(const shared_ptr<_Tp>& __a, const shared_ptr<_Up>& __b) noexcept
{ return !(__b < __a); }
template<typename _Tp>
[[__nodiscard__]] inline bool
operator<=(const shared_ptr<_Tp>& __a, nullptr_t) noexcept
{ return !(nullptr < __a); }
template<typename _Tp>
[[__nodiscard__]] inline bool
operator<=(nullptr_t, const shared_ptr<_Tp>& __a) noexcept
{ return !(__a < nullptr); }
template<typename _Tp, typename _Up>
[[__nodiscard__]] inline bool
operator>(const shared_ptr<_Tp>& __a, const shared_ptr<_Up>& __b) noexcept
{ return (__b < __a); }
template<typename _Tp>
[[__nodiscard__]] inline bool
operator>(const shared_ptr<_Tp>& __a, nullptr_t) noexcept
{ return nullptr < __a; }
template<typename _Tp>
[[__nodiscard__]] inline bool
operator>(nullptr_t, const shared_ptr<_Tp>& __a) noexcept
{ return __a < nullptr; }
template<typename _Tp, typename _Up>
[[__nodiscard__]] inline bool
operator>=(const shared_ptr<_Tp>& __a, const shared_ptr<_Up>& __b) noexcept
{ return !(__a < __b); }
template<typename _Tp>
[[__nodiscard__]] inline bool
operator>=(const shared_ptr<_Tp>& __a, nullptr_t) noexcept
{ return !(__a < nullptr); }
template<typename _Tp>
[[__nodiscard__]] inline bool
operator>=(nullptr_t, const shared_ptr<_Tp>& __a) noexcept
{ return !(nullptr < __a); }
template<typename _Tp>
inline void
swap(shared_ptr<_Tp>& __a, shared_ptr<_Tp>& __b) noexcept
{ __a.swap(__b); }
template<typename _Tp, typename _Up>
inline shared_ptr<_Tp>
static_pointer_cast(const shared_ptr<_Up>& __r) noexcept
{
using _Sp = shared_ptr<_Tp>;
return _Sp(__r, static_cast<typename _Sp::element_type*>(__r.get()));
}
template<typename _Tp, typename _Up>
inline shared_ptr<_Tp>
const_pointer_cast(const shared_ptr<_Up>& __r) noexcept
{
using _Sp = shared_ptr<_Tp>;
return _Sp(__r, const_cast<typename _Sp::element_type*>(__r.get()));
}
template<typename _Tp, typename _Up>
inline shared_ptr<_Tp>
dynamic_pointer_cast(const shared_ptr<_Up>& __r) noexcept
{
using _Sp = shared_ptr<_Tp>;
if (auto* __p = dynamic_cast<typename _Sp::element_type*>(__r.get()))
return _Sp(__r, __p);
return _Sp();
}
template<typename _Tp, typename _Up>
inline shared_ptr<_Tp>
reinterpret_pointer_cast(const shared_ptr<_Up>& __r) noexcept
{
using _Sp = shared_ptr<_Tp>;
return _Sp(__r, reinterpret_cast<typename _Sp::element_type*>(__r.get()));
}
template<typename _Tp>
class weak_ptr : public __weak_ptr<_Tp>
{
template<typename _Arg>
using _Constructible = typename enable_if<
is_constructible<__weak_ptr<_Tp>, _Arg>::value
>::type;
template<typename _Arg>
using _Assignable = typename enable_if<
is_assignable<__weak_ptr<_Tp>&, _Arg>::value, weak_ptr&
>::type;
public:
constexpr weak_ptr() noexcept = default;
template<typename _Yp,
typename = _Constructible<const shared_ptr<_Yp>&>>
weak_ptr(const shared_ptr<_Yp>& __r) noexcept
: __weak_ptr<_Tp>(__r) { }
weak_ptr(const weak_ptr&) noexcept = default;
template<typename _Yp, typename = _Constructible<const weak_ptr<_Yp>&>>
weak_ptr(const weak_ptr<_Yp>& __r) noexcept
: __weak_ptr<_Tp>(__r) { }
weak_ptr(weak_ptr&&) noexcept = default;
template<typename _Yp, typename = _Constructible<weak_ptr<_Yp>>>
weak_ptr(weak_ptr<_Yp>&& __r) noexcept
: __weak_ptr<_Tp>(std::move(__r)) { }
weak_ptr&
operator=(const weak_ptr& __r) noexcept = default;
template<typename _Yp>
_Assignable<const weak_ptr<_Yp>&>
operator=(const weak_ptr<_Yp>& __r) noexcept
{
this->__weak_ptr<_Tp>::operator=(__r);
return *this;
}
template<typename _Yp>
_Assignable<const shared_ptr<_Yp>&>
operator=(const shared_ptr<_Yp>& __r) noexcept
{
this->__weak_ptr<_Tp>::operator=(__r);
return *this;
}
weak_ptr&
operator=(weak_ptr&& __r) noexcept = default;
template<typename _Yp>
_Assignable<weak_ptr<_Yp>>
operator=(weak_ptr<_Yp>&& __r) noexcept
{
this->__weak_ptr<_Tp>::operator=(std::move(__r));
return *this;
}
shared_ptr<_Tp>
lock() const noexcept
{ return shared_ptr<_Tp>(*this, std::nothrow); }
};
template<typename _Tp>
weak_ptr(shared_ptr<_Tp>) -> weak_ptr<_Tp>;
template<typename _Tp>
inline void
swap(weak_ptr<_Tp>& __a, weak_ptr<_Tp>& __b) noexcept
{ __a.swap(__b); }
template<typename _Tp = void>
struct owner_less;
template<>
struct owner_less<void> : _Sp_owner_less<void, void>
{ };
template<typename _Tp>
struct owner_less<shared_ptr<_Tp>>
: public _Sp_owner_less<shared_ptr<_Tp>, weak_ptr<_Tp>>
{ };
template<typename _Tp>
struct owner_less<weak_ptr<_Tp>>
: public _Sp_owner_less<weak_ptr<_Tp>, shared_ptr<_Tp>>
{ };
template<typename _Tp>
class enable_shared_from_this
{
protected:
constexpr enable_shared_from_this() noexcept { }
enable_shared_from_this(const enable_shared_from_this&) noexcept { }
enable_shared_from_this&
operator=(const enable_shared_from_this&) noexcept
{ return *this; }
~enable_shared_from_this() { }
public:
shared_ptr<_Tp>
shared_from_this()
{ return shared_ptr<_Tp>(this->_M_weak_this); }
shared_ptr<const _Tp>
shared_from_this() const
{ return shared_ptr<const _Tp>(this->_M_weak_this); }
weak_ptr<_Tp>
weak_from_this() noexcept
{ return this->_M_weak_this; }
weak_ptr<const _Tp>
weak_from_this() const noexcept
{ return this->_M_weak_this; }
private:
template<typename _Tp1>
void
_M_weak_assign(_Tp1* __p, const __shared_count<>& __n) const noexcept
{ _M_weak_this._M_assign(__p, __n); }
friend const enable_shared_from_this*
__enable_shared_from_this_base(const __shared_count<>&,
const enable_shared_from_this* __p)
{ return __p; }
template<typename, _Lock_policy>
friend class __shared_ptr;
mutable weak_ptr<_Tp> _M_weak_this;
};
template<typename _Tp, typename _Alloc, typename... _Args>
inline shared_ptr<_NonArray<_Tp>>
allocate_shared(const _Alloc& __a, _Args&&... __args)
{
return shared_ptr<_Tp>(_Sp_alloc_shared_tag<_Alloc>{__a},
std::forward<_Args>(__args)...);
}
template<typename _Tp, typename... _Args>
inline shared_ptr<_NonArray<_Tp>>
make_shared(_Args&&... __args)
{
using _Alloc = allocator<void>;
_Alloc __a;
return shared_ptr<_Tp>(_Sp_alloc_shared_tag<_Alloc>{__a},
std::forward<_Args>(__args)...);
}
template<typename _Tp>
struct hash<shared_ptr<_Tp>>
: public __hash_base<size_t, shared_ptr<_Tp>>
{
size_t
operator()(const shared_ptr<_Tp>& __s) const noexcept
{
return std::hash<typename shared_ptr<_Tp>::element_type*>()(__s.get());
}
};
namespace __detail::__variant
{
template<typename> struct _Never_valueless_alt;
template<typename _Tp>
struct _Never_valueless_alt<std::shared_ptr<_Tp>>
: std::true_type
{ };
template<typename _Tp>
struct _Never_valueless_alt<std::weak_ptr<_Tp>>
: std::true_type
{ };
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
struct _Sp_locker
{
_Sp_locker(const _Sp_locker&) = delete;
_Sp_locker& operator=(const _Sp_locker&) = delete;
explicit
_Sp_locker(const void*) noexcept;
_Sp_locker(const void*, const void*) noexcept;
~_Sp_locker();
private:
unsigned char _M_key1;
unsigned char _M_key2;
};
template<typename _Tp, _Lock_policy _Lp>
inline bool
atomic_is_lock_free(const __shared_ptr<_Tp, _Lp>* __p)
{
return __gthread_active_p() == 0;
}
template<typename _Tp>
inline bool
atomic_is_lock_free(const shared_ptr<_Tp>* __p)
{ return std::atomic_is_lock_free<_Tp, __default_lock_policy>(__p); }
template<typename _Tp>
inline shared_ptr<_Tp>
atomic_load_explicit(const shared_ptr<_Tp>* __p, memory_order)
{
_Sp_locker __lock{__p};
return *__p;
}
template<typename _Tp>
inline shared_ptr<_Tp>
atomic_load(const shared_ptr<_Tp>* __p)
{ return std::atomic_load_explicit(__p, memory_order_seq_cst); }
template<typename _Tp, _Lock_policy _Lp>
inline __shared_ptr<_Tp, _Lp>
atomic_load_explicit(const __shared_ptr<_Tp, _Lp>* __p, memory_order)
{
_Sp_locker __lock{__p};
return *__p;
}
template<typename _Tp, _Lock_policy _Lp>
inline __shared_ptr<_Tp, _Lp>
atomic_load(const __shared_ptr<_Tp, _Lp>* __p)
{ return std::atomic_load_explicit(__p, memory_order_seq_cst); }
template<typename _Tp>
inline void
atomic_store_explicit(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r,
memory_order)
{
_Sp_locker __lock{__p};
__p->swap(__r);
}
template<typename _Tp>
inline void
atomic_store(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r)
{ std::atomic_store_explicit(__p, std::move(__r), memory_order_seq_cst); }
template<typename _Tp, _Lock_policy _Lp>
inline void
atomic_store_explicit(__shared_ptr<_Tp, _Lp>* __p,
__shared_ptr<_Tp, _Lp> __r,
memory_order)
{
_Sp_locker __lock{__p};
__p->swap(__r);
}
template<typename _Tp, _Lock_policy _Lp>
inline void
atomic_store(__shared_ptr<_Tp, _Lp>* __p, __shared_ptr<_Tp, _Lp> __r)
{ std::atomic_store_explicit(__p, std::move(__r), memory_order_seq_cst); }
template<typename _Tp>
inline shared_ptr<_Tp>
atomic_exchange_explicit(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r,
memory_order)
{
_Sp_locker __lock{__p};
__p->swap(__r);
return __r;
}
template<typename _Tp>
inline shared_ptr<_Tp>
atomic_exchange(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r)
{
return std::atomic_exchange_explicit(__p, std::move(__r),
memory_order_seq_cst);
}
template<typename _Tp, _Lock_policy _Lp>
inline __shared_ptr<_Tp, _Lp>
atomic_exchange_explicit(__shared_ptr<_Tp, _Lp>* __p,
__shared_ptr<_Tp, _Lp> __r,
memory_order)
{
_Sp_locker __lock{__p};
__p->swap(__r);
return __r;
}
template<typename _Tp, _Lock_policy _Lp>
inline __shared_ptr<_Tp, _Lp>
atomic_exchange(__shared_ptr<_Tp, _Lp>* __p, __shared_ptr<_Tp, _Lp> __r)
{
return std::atomic_exchange_explicit(__p, std::move(__r),
memory_order_seq_cst);
}
template<typename _Tp>
bool
atomic_compare_exchange_strong_explicit(shared_ptr<_Tp>* __p,
shared_ptr<_Tp>* __v,
shared_ptr<_Tp> __w,
memory_order,
memory_order)
{
shared_ptr<_Tp> __x;
_Sp_locker __lock{__p, __v};
owner_less<shared_ptr<_Tp>> __less;
if (*__p == *__v && !__less(*__p, *__v) && !__less(*__v, *__p))
{
__x = std::move(*__p);
*__p = std::move(__w);
return true;
}
__x = std::move(*__v);
*__v = *__p;
return false;
}
template<typename _Tp>
inline bool
atomic_compare_exchange_strong(shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v,
shared_ptr<_Tp> __w)
{
return std::atomic_compare_exchange_strong_explicit(__p, __v,
std::move(__w), memory_order_seq_cst, memory_order_seq_cst);
}
template<typename _Tp>
inline bool
atomic_compare_exchange_weak_explicit(shared_ptr<_Tp>* __p,
shared_ptr<_Tp>* __v,
shared_ptr<_Tp> __w,
memory_order __success,
memory_order __failure)
{
return std::atomic_compare_exchange_strong_explicit(__p, __v,
std::move(__w), __success, __failure);
}
template<typename _Tp>
inline bool
atomic_compare_exchange_weak(shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v,
shared_ptr<_Tp> __w)
{
return std::atomic_compare_exchange_weak_explicit(__p, __v,
std::move(__w), memory_order_seq_cst, memory_order_seq_cst);
}
template<typename _Tp, _Lock_policy _Lp>
bool
atomic_compare_exchange_strong_explicit(__shared_ptr<_Tp, _Lp>* __p,
__shared_ptr<_Tp, _Lp>* __v,
__shared_ptr<_Tp, _Lp> __w,
memory_order,
memory_order)
{
__shared_ptr<_Tp, _Lp> __x;
_Sp_locker __lock{__p, __v};
owner_less<__shared_ptr<_Tp, _Lp>> __less;
if (*__p == *__v && !__less(*__p, *__v) && !__less(*__v, *__p))
{
__x = std::move(*__p);
*__p = std::move(__w);
return true;
}
__x = std::move(*__v);
*__v = *__p;
return false;
}
template<typename _Tp, _Lock_policy _Lp>
inline bool
atomic_compare_exchange_strong(__shared_ptr<_Tp, _Lp>* __p,
__shared_ptr<_Tp, _Lp>* __v,
__shared_ptr<_Tp, _Lp> __w)
{
return std::atomic_compare_exchange_strong_explicit(__p, __v,
std::move(__w), memory_order_seq_cst, memory_order_seq_cst);
}
template<typename _Tp, _Lock_policy _Lp>
inline bool
atomic_compare_exchange_weak_explicit(__shared_ptr<_Tp, _Lp>* __p,
__shared_ptr<_Tp, _Lp>* __v,
__shared_ptr<_Tp, _Lp> __w,
memory_order __success,
memory_order __failure)
{
return std::atomic_compare_exchange_strong_explicit(__p, __v,
std::move(__w), __success, __failure);
}
template<typename _Tp, _Lock_policy _Lp>
inline bool
atomic_compare_exchange_weak(__shared_ptr<_Tp, _Lp>* __p,
__shared_ptr<_Tp, _Lp>* __v,
__shared_ptr<_Tp, _Lp> __w)
{
return std::atomic_compare_exchange_weak_explicit(__p, __v,
std::move(__w), memory_order_seq_cst, memory_order_seq_cst);
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Tp1>
struct auto_ptr_ref
{
_Tp1* _M_ptr;
explicit
auto_ptr_ref(_Tp1* __p): _M_ptr(__p) { }
} __attribute__ ((__deprecated__));
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Tp>
class auto_ptr
{
private:
_Tp* _M_ptr;
public:
typedef _Tp element_type;
explicit
auto_ptr(element_type* __p = 0) throw() : _M_ptr(__p) { }
auto_ptr(auto_ptr& __a) throw() : _M_ptr(__a.release()) { }
template<typename _Tp1>
auto_ptr(auto_ptr<_Tp1>& __a) throw() : _M_ptr(__a.release()) { }
auto_ptr&
operator=(auto_ptr& __a) throw()
{
reset(__a.release());
return *this;
}
template<typename _Tp1>
auto_ptr&
operator=(auto_ptr<_Tp1>& __a) throw()
{
reset(__a.release());
return *this;
}
~auto_ptr() { delete _M_ptr; }
element_type&
operator*() const throw()
{
do { if (std::__is_constant_evaluated() && !bool(_M_ptr != 0)) __builtin_unreachable(); } while (false);
return *_M_ptr;
}
element_type*
operator->() const throw()
{
do { if (std::__is_constant_evaluated() && !bool(_M_ptr != 0)) __builtin_unreachable(); } while (false);
return _M_ptr;
}
element_type*
get() const throw() { return _M_ptr; }
element_type*
release() throw()
{
element_type* __tmp = _M_ptr;
_M_ptr = 0;
return __tmp;
}
void
reset(element_type* __p = 0) throw()
{
if (__p != _M_ptr)
{
delete _M_ptr;
_M_ptr = __p;
}
}
auto_ptr(auto_ptr_ref<element_type> __ref) throw()
: _M_ptr(__ref._M_ptr) { }
auto_ptr&
operator=(auto_ptr_ref<element_type> __ref) throw()
{
if (__ref._M_ptr != this->get())
{
delete _M_ptr;
_M_ptr = __ref._M_ptr;
}
return *this;
}
template<typename _Tp1>
operator auto_ptr_ref<_Tp1>() throw()
{ return auto_ptr_ref<_Tp1>(this->release()); }
template<typename _Tp1>
operator auto_ptr<_Tp1>() throw()
{ return auto_ptr<_Tp1>(this->release()); }
} __attribute__ ((__deprecated__ ("use '" "std::unique_ptr" "' instead")));
template<>
class auto_ptr<void>
{
public:
typedef void element_type;
} __attribute__ ((__deprecated__));
template<_Lock_policy _Lp>
template<typename _Tp>
inline
__shared_count<_Lp>::__shared_count(std::auto_ptr<_Tp>&& __r)
: _M_pi(new _Sp_counted_ptr<_Tp*, _Lp>(__r.get()))
{ __r.release(); }
template<typename _Tp, _Lock_policy _Lp>
template<typename _Tp1, typename>
inline
__shared_ptr<_Tp, _Lp>::__shared_ptr(std::auto_ptr<_Tp1>&& __r)
: _M_ptr(__r.get()), _M_refcount()
{
static_assert( sizeof(_Tp1) > 0, "incomplete type" );
_Tp1* __tmp = __r.get();
_M_refcount = __shared_count<_Lp>(std::move(__r));
_M_enable_shared_from_this_with(__tmp);
}
template<typename _Tp>
template<typename _Tp1, typename>
inline
shared_ptr<_Tp>::shared_ptr(std::auto_ptr<_Tp1>&& __r)
: __shared_ptr<_Tp>(std::move(__r)) { }
template<typename _Tp, typename _Dp>
template<typename _Up, typename>
inline
unique_ptr<_Tp, _Dp>::unique_ptr(auto_ptr<_Up>&& __u) noexcept
: _M_t(__u.release(), deleter_type()) { }
#pragma GCC diagnostic pop
}
namespace std __attribute__ ((__visibility__ ("default")))
{
enum class pointer_safety { relaxed, preferred, strict };
inline void
declare_reachable(void*) { }
template <typename _Tp>
inline _Tp*
undeclare_reachable(_Tp* __p) { return __p; }
inline void
declare_no_pointers(char*, size_t) { }
inline void
undeclare_no_pointers(char*, size_t) { }
inline pointer_safety
get_pointer_safety() noexcept { return pointer_safety::relaxed; }
}
namespace __pstl
{
namespace execution
{
inline namespace v1
{
class sequenced_policy
{
public:
static constexpr std::false_type
__allow_unsequenced()
{
return std::false_type{};
}
static constexpr std::false_type
__allow_vector()
{
return std::false_type{};
}
static constexpr std::false_type
__allow_parallel()
{
return std::false_type{};
}
};
class parallel_policy
{
public:
static constexpr std::false_type
__allow_unsequenced()
{
return std::false_type{};
}
static constexpr std::false_type
__allow_vector()
{
return std::false_type{};
}
static constexpr std::true_type
__allow_parallel()
{
return std::true_type{};
}
};
class parallel_unsequenced_policy
{
public:
static constexpr std::true_type
__allow_unsequenced()
{
return std::true_type{};
}
static constexpr std::true_type
__allow_vector()
{
return std::true_type{};
}
static constexpr std::true_type
__allow_parallel()
{
return std::true_type{};
}
};
class unsequenced_policy
{
public:
static constexpr std::true_type
__allow_unsequenced()
{
return std::true_type{};
}
static constexpr std::true_type
__allow_vector()
{
return std::true_type{};
}
static constexpr std::false_type
__allow_parallel()
{
return std::false_type{};
}
};
inline constexpr sequenced_policy seq{};
inline constexpr parallel_policy par{};
inline constexpr parallel_unsequenced_policy par_unseq{};
inline constexpr unsequenced_policy unseq{};
template <class _Tp>
struct is_execution_policy : std::false_type
{
};
template <>
struct is_execution_policy<__pstl::execution::sequenced_policy> : std::true_type
{
};
template <>
struct is_execution_policy<__pstl::execution::parallel_policy> : std::true_type
{
};
template <>
struct is_execution_policy<__pstl::execution::parallel_unsequenced_policy> : std::true_type
{
};
template <>
struct is_execution_policy<__pstl::execution::unsequenced_policy> : std::true_type
{
};
template <class _Tp>
constexpr bool is_execution_policy_v = __pstl::execution::is_execution_policy<_Tp>::value;
}
}
namespace __internal
{
template <class _ExecPolicy, class _Tp>
using __enable_if_execution_policy =
typename std::enable_if<__pstl::execution::is_execution_policy<std::__remove_cvref_t<_ExecPolicy>>::value,
_Tp>::type;
}
}
namespace std
{
template <class _ExecutionPolicy, class _InputIterator, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_copy(_ExecutionPolicy&& __exec, _InputIterator __first, _InputIterator __last, _ForwardIterator __result);
template <class _ExecutionPolicy, class _InputIterator, class _Size, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_copy_n(_ExecutionPolicy&& __exec, _InputIterator __first, _Size __n, _ForwardIterator __result);
template <class _ExecutionPolicy, class _InputIterator, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_move(_ExecutionPolicy&& __exec, _InputIterator __first, _InputIterator __last, _ForwardIterator __result);
template <class _ExecutionPolicy, class _InputIterator, class _Size, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_move_n(_ExecutionPolicy&& __exec, _InputIterator __first, _Size __n, _ForwardIterator __result);
template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
uninitialized_fill(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value);
template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_fill_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n, const _Tp& __value);
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
destroy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
destroy_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n);
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
uninitialized_default_construct(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_default_construct_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n);
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
uninitialized_value_construct(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_value_construct_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n);
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace rel_ops
{
template <class _Tp>
inline bool
operator!=(const _Tp& __x, const _Tp& __y)
{ return !(__x == __y); }
template <class _Tp>
inline bool
operator>(const _Tp& __x, const _Tp& __y)
{ return __y < __x; }
template <class _Tp>
inline bool
operator<=(const _Tp& __x, const _Tp& __y)
{ return !(__y < __x); }
template <class _Tp>
inline bool
operator>=(const _Tp& __x, const _Tp& __y)
{ return !(__x < __y); }
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template <typename _Tp, typename _Up = _Tp>
inline _Tp
exchange(_Tp& __obj, _Up&& __new_val)
noexcept(__and_<is_nothrow_move_constructible<_Tp>,
is_nothrow_assignable<_Tp&, _Up>>::value)
{ return std::__exchange(__obj, std::forward<_Up>(__new_val)); }
template<typename _Tp>
[[nodiscard]]
constexpr add_const_t<_Tp>&
as_const(_Tp& __t) noexcept
{ return __t; }
template<typename _Tp>
void as_const(const _Tp&&) = delete;
}
namespace hwy {
using AllocPtr = void* (*)(void* opaque, size_t bytes);
using FreePtr = void (*)(void* opaque, void* memory);
void* AllocateAlignedBytes(size_t payload_size,
AllocPtr alloc_ptr, void* opaque_ptr);
void FreeAlignedBytes(const void* aligned_pointer,
FreePtr free_ptr, void* opaque_ptr);
class AlignedDeleter {
public:
AlignedDeleter() : free_(nullptr), opaque_ptr_(nullptr) {}
AlignedDeleter(FreePtr free_ptr, void* opaque_ptr)
: free_(free_ptr), opaque_ptr_(opaque_ptr) {}
template <typename T>
void operator()(T* aligned_pointer) const {
return DeleteAlignedArray(aligned_pointer, free_, opaque_ptr_,
TypedArrayDeleter<T>);
}
private:
template <typename T>
static void TypedArrayDeleter(void* ptr, size_t size_in_bytes) {
size_t elems = size_in_bytes / sizeof(T);
for (size_t i = 0; i < elems; i++) {
(static_cast<T*>(ptr) + i)->~T();
}
}
using ArrayDeleter = void (*)(void* t_ptr, size_t t_size);
static void DeleteAlignedArray(void* aligned_pointer,
FreePtr free_ptr,
void* opaque_ptr,
ArrayDeleter deleter);
FreePtr free_;
void* opaque_ptr_;
};
template <typename T>
using AlignedUniquePtr = std::unique_ptr<T, AlignedDeleter>;
template <typename T, typename... Args>
AlignedUniquePtr<T> MakeUniqueAlignedWithAlloc(AllocPtr alloc, FreePtr free,
void* opaque, Args&&... args) {
T* ptr = static_cast<T*>(AllocateAlignedBytes(sizeof(T), alloc, opaque));
return AlignedUniquePtr<T>(new (ptr) T(std::forward<Args>(args)...),
AlignedDeleter(free, opaque));
}
template <typename T, typename... Args>
AlignedUniquePtr<T> MakeUniqueAligned(Args&&... args) {
T* ptr = static_cast<T*>(AllocateAlignedBytes(
sizeof(T), nullptr, nullptr));
return AlignedUniquePtr<T>(new (ptr) T(std::forward<Args>(args)...),
AlignedDeleter());
}
namespace detail {
static inline constexpr size_t ShiftCount(size_t n) {
return (n <= 1) ? 0 : 1 + ShiftCount(n / 2);
}
template <typename T>
T* AllocateAlignedItems(size_t items, AllocPtr alloc_ptr, void* opaque_ptr) {
constexpr size_t size = sizeof(T);
constexpr bool is_pow2 = (size & (size - 1)) == 0;
constexpr size_t bits = ShiftCount(size);
static_assert(!is_pow2 || (1ull << bits) == size, "ShiftCount is incorrect");
const size_t bytes = is_pow2 ? items << bits : items * size;
const size_t check = is_pow2 ? bytes >> bits : bytes / size;
if (check != items) {
return nullptr;
}
return static_cast<T*>(AllocateAlignedBytes(bytes, alloc_ptr, opaque_ptr));
}
}
template <typename T, typename... Args>
AlignedUniquePtr<T[]> MakeUniqueAlignedArrayWithAlloc(
size_t items, AllocPtr alloc, FreePtr free, void* opaque, Args&&... args) {
T* ptr = detail::AllocateAlignedItems<T>(items, alloc, opaque);
if (ptr != nullptr) {
for (size_t i = 0; i < items; i++) {
new (ptr + i) T(std::forward<Args>(args)...);
}
}
return AlignedUniquePtr<T[]>(ptr, AlignedDeleter(free, opaque));
}
template <typename T, typename... Args>
AlignedUniquePtr<T[]> MakeUniqueAlignedArray(size_t items, Args&&... args) {
return MakeUniqueAlignedArrayWithAlloc<T, Args...>(
items, nullptr, nullptr, nullptr, std::forward<Args>(args)...);
}
class AlignedFreer {
public:
static void DoNothing(void* , void* ) {}
AlignedFreer() : free_(nullptr), opaque_ptr_(nullptr) {}
AlignedFreer(FreePtr free_ptr, void* opaque_ptr)
: free_(free_ptr), opaque_ptr_(opaque_ptr) {}
template <typename T>
void operator()(T* aligned_pointer) const {
FreeAlignedBytes(aligned_pointer, free_, opaque_ptr_);
}
private:
FreePtr free_;
void* opaque_ptr_;
};
template <typename T>
using AlignedFreeUniquePtr = std::unique_ptr<T, AlignedFreer>;
template <typename T>
AlignedFreeUniquePtr<T[]> AllocateAligned(const size_t items, AllocPtr alloc,
FreePtr free, void* opaque) {
return AlignedFreeUniquePtr<T[]>(
detail::AllocateAlignedItems<T>(items, alloc, opaque),
AlignedFreer(free, opaque));
}
template <typename T>
AlignedFreeUniquePtr<T[]> AllocateAligned(const size_t items) {
return AllocateAligned<T>(items, nullptr, nullptr, nullptr);
}
}
namespace __gnu_cxx __attribute__ ((__visibility__ ("default")))
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wstringop-overflow"
#pragma GCC diagnostic ignored "-Wstringop-overread"
#pragma GCC diagnostic ignored "-Warray-bounds"
template<typename _CharT>
struct _Char_types
{
typedef unsigned long int_type;
typedef std::streampos pos_type;
typedef std::streamoff off_type;
typedef std::mbstate_t state_type;
};
template<typename _CharT>
struct char_traits
{
typedef _CharT char_type;
typedef typename _Char_types<_CharT>::int_type int_type;
typedef typename _Char_types<_CharT>::pos_type pos_type;
typedef typename _Char_types<_CharT>::off_type off_type;
typedef typename _Char_types<_CharT>::state_type state_type;
static constexpr void
assign(char_type& __c1, const char_type& __c2)
{
__c1 = __c2;
}
static constexpr bool
eq(const char_type& __c1, const char_type& __c2)
{ return __c1 == __c2; }
static constexpr bool
lt(const char_type& __c1, const char_type& __c2)
{ return __c1 < __c2; }
static constexpr int
compare(const char_type* __s1, const char_type* __s2, std::size_t __n);
static constexpr std::size_t
length(const char_type* __s);
static constexpr const char_type*
find(const char_type* __s, std::size_t __n, const char_type& __a);
static char_type*
move(char_type* __s1, const char_type* __s2, std::size_t __n);
static char_type*
copy(char_type* __s1, const char_type* __s2, std::size_t __n);
static char_type*
assign(char_type* __s, std::size_t __n, char_type __a);
static constexpr char_type
to_char_type(const int_type& __c)
{ return static_cast<char_type>(__c); }
static constexpr int_type
to_int_type(const char_type& __c)
{ return static_cast<int_type>(__c); }
static constexpr bool
eq_int_type(const int_type& __c1, const int_type& __c2)
{ return __c1 == __c2; }
static constexpr int_type
eof()
{ return static_cast<int_type>(-1); }
static constexpr int_type
not_eof(const int_type& __c)
{ return !eq_int_type(__c, eof()) ? __c : to_int_type(char_type()); }
};
template<typename _CharT>
constexpr int
char_traits<_CharT>::
compare(const char_type* __s1, const char_type* __s2, std::size_t __n)
{
for (std::size_t __i = 0; __i < __n; ++__i)
if (lt(__s1[__i], __s2[__i]))
return -1;
else if (lt(__s2[__i], __s1[__i]))
return 1;
return 0;
}
template<typename _CharT>
constexpr std::size_t
char_traits<_CharT>::
length(const char_type* __p)
{
std::size_t __i = 0;
while (!eq(__p[__i], char_type()))
++__i;
return __i;
}
template<typename _CharT>
constexpr const typename char_traits<_CharT>::char_type*
char_traits<_CharT>::
find(const char_type* __s, std::size_t __n, const char_type& __a)
{
for (std::size_t __i = 0; __i < __n; ++__i)
if (eq(__s[__i], __a))
return __s + __i;
return 0;
}
template<typename _CharT>
typename char_traits<_CharT>::char_type*
char_traits<_CharT>::
move(char_type* __s1, const char_type* __s2, std::size_t __n)
{
if (__n == 0)
return __s1;
__builtin_memmove(__s1, __s2, __n * sizeof(char_type));
return __s1;
}
template<typename _CharT>
typename char_traits<_CharT>::char_type*
char_traits<_CharT>::
copy(char_type* __s1, const char_type* __s2, std::size_t __n)
{
if (__n == 0)
return __s1;
__builtin_memcpy(__s1, __s2, __n * sizeof(char_type));
return __s1;
}
template<typename _CharT>
typename char_traits<_CharT>::char_type*
char_traits<_CharT>::
assign(char_type* __s, std::size_t __n, char_type __a)
{
if constexpr (sizeof(_CharT) == 1 && __is_trivial(_CharT))
{
if (__n)
{
unsigned char __c;
__builtin_memcpy(&__c, __builtin_addressof(__a), 1);
__builtin_memset(__s, __c, __n);
}
}
else
{
for (std::size_t __i = 0; __i < __n; ++__i)
__s[__i] = __a;
}
return __s;
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _CharT>
struct char_traits : public __gnu_cxx::char_traits<_CharT>
{ };
template<>
struct char_traits<char>
{
typedef char char_type;
typedef int int_type;
typedef streampos pos_type;
typedef streamoff off_type;
typedef mbstate_t state_type;
static constexpr void
assign(char_type& __c1, const char_type& __c2) noexcept
{
__c1 = __c2;
}
static constexpr bool
eq(const char_type& __c1, const char_type& __c2) noexcept
{ return __c1 == __c2; }
static constexpr bool
lt(const char_type& __c1, const char_type& __c2) noexcept
{
return (static_cast<unsigned char>(__c1)
< static_cast<unsigned char>(__c2));
}
static constexpr int
compare(const char_type* __s1, const char_type* __s2, size_t __n)
{
if (__n == 0)
return 0;
if (std::__is_constant_evaluated())
{
for (size_t __i = 0; __i < __n; ++__i)
if (lt(__s1[__i], __s2[__i]))
return -1;
else if (lt(__s2[__i], __s1[__i]))
return 1;
return 0;
}
return __builtin_memcmp(__s1, __s2, __n);
}
static constexpr size_t
length(const char_type* __s)
{
if (std::__is_constant_evaluated())
return __gnu_cxx::char_traits<char_type>::length(__s);
return __builtin_strlen(__s);
}
static constexpr const char_type*
find(const char_type* __s, size_t __n, const char_type& __a)
{
if (__n == 0)
return 0;
if (std::__is_constant_evaluated())
return __gnu_cxx::char_traits<char_type>::find(__s, __n, __a);
return static_cast<const char_type*>(__builtin_memchr(__s, __a, __n));
}
static char_type*
move(char_type* __s1, const char_type* __s2, size_t __n)
{
if (__n == 0)
return __s1;
return static_cast<char_type*>(__builtin_memmove(__s1, __s2, __n));
}
static char_type*
copy(char_type* __s1, const char_type* __s2, size_t __n)
{
if (__n == 0)
return __s1;
return static_cast<char_type*>(__builtin_memcpy(__s1, __s2, __n));
}
static char_type*
assign(char_type* __s, size_t __n, char_type __a)
{
if (__n == 0)
return __s;
return static_cast<char_type*>(__builtin_memset(__s, __a, __n));
}
static constexpr char_type
to_char_type(const int_type& __c) noexcept
{ return static_cast<char_type>(__c); }
static constexpr int_type
to_int_type(const char_type& __c) noexcept
{ return static_cast<int_type>(static_cast<unsigned char>(__c)); }
static constexpr bool
eq_int_type(const int_type& __c1, const int_type& __c2) noexcept
{ return __c1 == __c2; }
static constexpr int_type
eof() noexcept
{ return static_cast<int_type>(-1); }
static constexpr int_type
not_eof(const int_type& __c) noexcept
{ return (__c == eof()) ? 0 : __c; }
};
template<>
struct char_traits<wchar_t>
{
typedef wchar_t char_type;
typedef wint_t int_type;
typedef streamoff off_type;
typedef wstreampos pos_type;
typedef mbstate_t state_type;
static constexpr void
assign(char_type& __c1, const char_type& __c2) noexcept
{
__c1 = __c2;
}
static constexpr bool
eq(const char_type& __c1, const char_type& __c2) noexcept
{ return __c1 == __c2; }
static constexpr bool
lt(const char_type& __c1, const char_type& __c2) noexcept
{ return __c1 < __c2; }
static constexpr int
compare(const char_type* __s1, const char_type* __s2, size_t __n)
{
if (__n == 0)
return 0;
if (std::__is_constant_evaluated())
return __gnu_cxx::char_traits<char_type>::compare(__s1, __s2, __n);
return wmemcmp(__s1, __s2, __n);
}
static constexpr size_t
length(const char_type* __s)
{
if (std::__is_constant_evaluated())
return __gnu_cxx::char_traits<char_type>::length(__s);
return wcslen(__s);
}
static constexpr const char_type*
find(const char_type* __s, size_t __n, const char_type& __a)
{
if (__n == 0)
return 0;
if (std::__is_constant_evaluated())
return __gnu_cxx::char_traits<char_type>::find(__s, __n, __a);
return wmemchr(__s, __a, __n);
}
static char_type*
move(char_type* __s1, const char_type* __s2, size_t __n)
{
if (__n == 0)
return __s1;
return wmemmove(__s1, __s2, __n);
}
static char_type*
copy(char_type* __s1, const char_type* __s2, size_t __n)
{
if (__n == 0)
return __s1;
return wmemcpy(__s1, __s2, __n);
}
static char_type*
assign(char_type* __s, size_t __n, char_type __a)
{
if (__n == 0)
return __s;
return wmemset(__s, __a, __n);
}
static constexpr char_type
to_char_type(const int_type& __c) noexcept
{ return char_type(__c); }
static constexpr int_type
to_int_type(const char_type& __c) noexcept
{ return int_type(__c); }
static constexpr bool
eq_int_type(const int_type& __c1, const int_type& __c2) noexcept
{ return __c1 == __c2; }
static constexpr int_type
eof() noexcept
{ return static_cast<int_type>((0xffffffffu)); }
static constexpr int_type
not_eof(const int_type& __c) noexcept
{ return eq_int_type(__c, eof()) ? 0 : __c; }
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<>
struct char_traits<char16_t>
{
typedef char16_t char_type;
typedef short unsigned int int_type;
typedef streamoff off_type;
typedef u16streampos pos_type;
typedef mbstate_t state_type;
static constexpr void
assign(char_type& __c1, const char_type& __c2) noexcept
{
__c1 = __c2;
}
static constexpr bool
eq(const char_type& __c1, const char_type& __c2) noexcept
{ return __c1 == __c2; }
static constexpr bool
lt(const char_type& __c1, const char_type& __c2) noexcept
{ return __c1 < __c2; }
static constexpr int
compare(const char_type* __s1, const char_type* __s2, size_t __n)
{
for (size_t __i = 0; __i < __n; ++__i)
if (lt(__s1[__i], __s2[__i]))
return -1;
else if (lt(__s2[__i], __s1[__i]))
return 1;
return 0;
}
static constexpr size_t
length(const char_type* __s)
{
size_t __i = 0;
while (!eq(__s[__i], char_type()))
++__i;
return __i;
}
static constexpr const char_type*
find(const char_type* __s, size_t __n, const char_type& __a)
{
for (size_t __i = 0; __i < __n; ++__i)
if (eq(__s[__i], __a))
return __s + __i;
return 0;
}
static char_type*
move(char_type* __s1, const char_type* __s2, size_t __n)
{
if (__n == 0)
return __s1;
return (static_cast<char_type*>
(__builtin_memmove(__s1, __s2, __n * sizeof(char_type))));
}
static char_type*
copy(char_type* __s1, const char_type* __s2, size_t __n)
{
if (__n == 0)
return __s1;
return (static_cast<char_type*>
(__builtin_memcpy(__s1, __s2, __n * sizeof(char_type))));
}
static char_type*
assign(char_type* __s, size_t __n, char_type __a)
{
for (size_t __i = 0; __i < __n; ++__i)
assign(__s[__i], __a);
return __s;
}
static constexpr char_type
to_char_type(const int_type& __c) noexcept
{ return char_type(__c); }
static constexpr bool
eq_int_type(const int_type& __c1, const int_type& __c2) noexcept
{ return __c1 == __c2; }
static constexpr int_type
to_int_type(const char_type& __c) noexcept
{ return __c == eof() ? int_type(0xfffd) : int_type(__c); }
static constexpr int_type
eof() noexcept
{ return static_cast<int_type>(-1); }
static constexpr int_type
not_eof(const int_type& __c) noexcept
{ return eq_int_type(__c, eof()) ? 0 : __c; }
};
template<>
struct char_traits<char32_t>
{
typedef char32_t char_type;
typedef unsigned int int_type;
typedef streamoff off_type;
typedef u32streampos pos_type;
typedef mbstate_t state_type;
static constexpr void
assign(char_type& __c1, const char_type& __c2) noexcept
{
__c1 = __c2;
}
static constexpr bool
eq(const char_type& __c1, const char_type& __c2) noexcept
{ return __c1 == __c2; }
static constexpr bool
lt(const char_type& __c1, const char_type& __c2) noexcept
{ return __c1 < __c2; }
static constexpr int
compare(const char_type* __s1, const char_type* __s2, size_t __n)
{
for (size_t __i = 0; __i < __n; ++__i)
if (lt(__s1[__i], __s2[__i]))
return -1;
else if (lt(__s2[__i], __s1[__i]))
return 1;
return 0;
}
static constexpr size_t
length(const char_type* __s)
{
size_t __i = 0;
while (!eq(__s[__i], char_type()))
++__i;
return __i;
}
static constexpr const char_type*
find(const char_type* __s, size_t __n, const char_type& __a)
{
for (size_t __i = 0; __i < __n; ++__i)
if (eq(__s[__i], __a))
return __s + __i;
return 0;
}
static char_type*
move(char_type* __s1, const char_type* __s2, size_t __n)
{
if (__n == 0)
return __s1;
return (static_cast<char_type*>
(__builtin_memmove(__s1, __s2, __n * sizeof(char_type))));
}
static char_type*
copy(char_type* __s1, const char_type* __s2, size_t __n)
{
if (__n == 0)
return __s1;
return (static_cast<char_type*>
(__builtin_memcpy(__s1, __s2, __n * sizeof(char_type))));
}
static char_type*
assign(char_type* __s, size_t __n, char_type __a)
{
for (size_t __i = 0; __i < __n; ++__i)
assign(__s[__i], __a);
return __s;
}
static constexpr char_type
to_char_type(const int_type& __c) noexcept
{ return char_type(__c); }
static constexpr int_type
to_int_type(const char_type& __c) noexcept
{ return int_type(__c); }
static constexpr bool
eq_int_type(const int_type& __c1, const int_type& __c2) noexcept
{ return __c1 == __c2; }
static constexpr int_type
eof() noexcept
{ return static_cast<int_type>(-1); }
static constexpr int_type
not_eof(const int_type& __c) noexcept
{ return eq_int_type(__c, eof()) ? 0 : __c; }
};
#pragma GCC diagnostic pop
}
extern "C" {
struct lconv
{
char *decimal_point;
char *thousands_sep;
char *grouping;
char *int_curr_symbol;
char *currency_symbol;
char *mon_decimal_point;
char *mon_thousands_sep;
char *mon_grouping;
char *positive_sign;
char *negative_sign;
char int_frac_digits;
char frac_digits;
char p_cs_precedes;
char p_sep_by_space;
char n_cs_precedes;
char n_sep_by_space;
char p_sign_posn;
char n_sign_posn;
char int_p_cs_precedes;
char int_p_sep_by_space;
char int_n_cs_precedes;
char int_n_sep_by_space;
char int_p_sign_posn;
char int_n_sign_posn;
};
extern char *setlocale (int __category, const char *__locale) noexcept (true);
extern struct lconv *localeconv (void) noexcept (true);
extern locale_t newlocale (int __category_mask, const char *__locale,
locale_t __base) noexcept (true);
extern locale_t duplocale (locale_t __dataset) noexcept (true);
extern void freelocale (locale_t __dataset) noexcept (true);
extern locale_t uselocale (locale_t __dataset) noexcept (true);
}
namespace std
{
using ::lconv;
using ::setlocale;
using ::localeconv;
}
namespace __gnu_cxx __attribute__ ((__visibility__ ("default")))
{
extern "C" __typeof(uselocale) __uselocale;
}
namespace std __attribute__ ((__visibility__ ("default")))
{
typedef __locale_t __c_locale;
inline int
__convert_from_v(const __c_locale& __cloc __attribute__ ((__unused__)),
char* __out,
const int __size __attribute__ ((__unused__)),
const char* __fmt, ...)
{
__c_locale __old = __gnu_cxx::__uselocale(__cloc);
__builtin_va_list __args;
__builtin_va_start(__args, __fmt);
const int __ret = __builtin_vsnprintf(__out, __size, __fmt, __args);
__builtin_va_end(__args);
__gnu_cxx::__uselocale(__old);
return __ret;
}
}
extern "C" {
enum
{
_ISupper = ((0) < 8 ? ((1 << (0)) << 8) : ((1 << (0)) >> 8)),
_ISlower = ((1) < 8 ? ((1 << (1)) << 8) : ((1 << (1)) >> 8)),
_ISalpha = ((2) < 8 ? ((1 << (2)) << 8) : ((1 << (2)) >> 8)),
_ISdigit = ((3) < 8 ? ((1 << (3)) << 8) : ((1 << (3)) >> 8)),
_ISxdigit = ((4) < 8 ? ((1 << (4)) << 8) : ((1 << (4)) >> 8)),
_ISspace = ((5) < 8 ? ((1 << (5)) << 8) : ((1 << (5)) >> 8)),
_ISprint = ((6) < 8 ? ((1 << (6)) << 8) : ((1 << (6)) >> 8)),
_ISgraph = ((7) < 8 ? ((1 << (7)) << 8) : ((1 << (7)) >> 8)),
_ISblank = ((8) < 8 ? ((1 << (8)) << 8) : ((1 << (8)) >> 8)),
_IScntrl = ((9) < 8 ? ((1 << (9)) << 8) : ((1 << (9)) >> 8)),
_ISpunct = ((10) < 8 ? ((1 << (10)) << 8) : ((1 << (10)) >> 8)),
_ISalnum = ((11) < 8 ? ((1 << (11)) << 8) : ((1 << (11)) >> 8))
};
extern const unsigned short int **__ctype_b_loc (void)
noexcept (true) __attribute__ ((__const__));
extern const __int32_t **__ctype_tolower_loc (void)
noexcept (true) __attribute__ ((__const__));
extern const __int32_t **__ctype_toupper_loc (void)
noexcept (true) __attribute__ ((__const__));
extern int isalnum (int) noexcept (true);
extern int isalpha (int) noexcept (true);
extern int iscntrl (int) noexcept (true);
extern int isdigit (int) noexcept (true);
extern int islower (int) noexcept (true);
extern int isgraph (int) noexcept (true);
extern int isprint (int) noexcept (true);
extern int ispunct (int) noexcept (true);
extern int isspace (int) noexcept (true);
extern int isupper (int) noexcept (true);
extern int isxdigit (int) noexcept (true);
extern int tolower (int __c) noexcept (true);
extern int toupper (int __c) noexcept (true);
extern int isblank (int) noexcept (true);
extern int isctype (int __c, int __mask) noexcept (true);
extern int isascii (int __c) noexcept (true);
extern int toascii (int __c) noexcept (true);
extern int _toupper (int) noexcept (true);
extern int _tolower (int) noexcept (true);
extern int isalnum_l (int, locale_t) noexcept (true);
extern int isalpha_l (int, locale_t) noexcept (true);
extern int iscntrl_l (int, locale_t) noexcept (true);
extern int isdigit_l (int, locale_t) noexcept (true);
extern int islower_l (int, locale_t) noexcept (true);
extern int isgraph_l (int, locale_t) noexcept (true);
extern int isprint_l (int, locale_t) noexcept (true);
extern int ispunct_l (int, locale_t) noexcept (true);
extern int isspace_l (int, locale_t) noexcept (true);
extern int isupper_l (int, locale_t) noexcept (true);
extern int isxdigit_l (int, locale_t) noexcept (true);
extern int isblank_l (int, locale_t) noexcept (true);
extern int __tolower_l (int __c, locale_t __l) noexcept (true);
extern int tolower_l (int __c, locale_t __l) noexcept (true);
extern int __toupper_l (int __c, locale_t __l) noexcept (true);
extern int toupper_l (int __c, locale_t __l) noexcept (true);
}
namespace std
{
using ::isalnum;
using ::isalpha;
using ::iscntrl;
using ::isdigit;
using ::isgraph;
using ::islower;
using ::isprint;
using ::ispunct;
using ::isspace;
using ::isupper;
using ::isxdigit;
using ::tolower;
using ::toupper;
}
namespace std
{
using ::isblank;
}
namespace std __attribute__ ((__visibility__ ("default")))
{
class locale;
template<typename _Facet>
bool
has_facet(const locale&) throw();
template<typename _Facet>
const _Facet&
use_facet(const locale&);
template<typename _CharT>
bool
isspace(_CharT, const locale&);
template<typename _CharT>
bool
isprint(_CharT, const locale&);
template<typename _CharT>
bool
iscntrl(_CharT, const locale&);
template<typename _CharT>
bool
isupper(_CharT, const locale&);
template<typename _CharT>
bool
islower(_CharT, const locale&);
template<typename _CharT>
bool
isalpha(_CharT, const locale&);
template<typename _CharT>
bool
isdigit(_CharT, const locale&);
template<typename _CharT>
bool
ispunct(_CharT, const locale&);
template<typename _CharT>
bool
isxdigit(_CharT, const locale&);
template<typename _CharT>
bool
isalnum(_CharT, const locale&);
template<typename _CharT>
bool
isgraph(_CharT, const locale&);
template<typename _CharT>
bool
isblank(_CharT, const locale&);
template<typename _CharT>
_CharT
toupper(_CharT, const locale&);
template<typename _CharT>
_CharT
tolower(_CharT, const locale&);
struct ctype_base;
template<typename _CharT>
class ctype;
template<> class ctype<char>;
template<> class ctype<wchar_t>;
template<typename _CharT>
class ctype_byname;
class codecvt_base;
template<typename _InternT, typename _ExternT, typename _StateT>
class codecvt;
template<> class codecvt<char, char, mbstate_t>;
template<> class codecvt<wchar_t, char, mbstate_t>;
template<> class codecvt<char16_t, char, mbstate_t>;
template<> class codecvt<char32_t, char, mbstate_t>;
template<typename _InternT, typename _ExternT, typename _StateT>
class codecvt_byname;
template<typename _CharT, typename _InIter = istreambuf_iterator<_CharT> >
class num_get;
template<typename _CharT, typename _OutIter = ostreambuf_iterator<_CharT> >
class num_put;
namespace __cxx11 {
template<typename _CharT> class numpunct;
template<typename _CharT> class numpunct_byname;
}
namespace __cxx11 {
template<typename _CharT>
class collate;
template<typename _CharT>
class collate_byname;
}
class time_base;
namespace __cxx11 {
template<typename _CharT, typename _InIter = istreambuf_iterator<_CharT> >
class time_get;
template<typename _CharT, typename _InIter = istreambuf_iterator<_CharT> >
class time_get_byname;
}
template<typename _CharT, typename _OutIter = ostreambuf_iterator<_CharT> >
class time_put;
template<typename _CharT, typename _OutIter = ostreambuf_iterator<_CharT> >
class time_put_byname;
class money_base;
namespace __cxx11 {
template<typename _CharT, typename _InIter = istreambuf_iterator<_CharT> >
class money_get;
template<typename _CharT, typename _OutIter = ostreambuf_iterator<_CharT> >
class money_put;
}
namespace __cxx11 {
template<typename _CharT, bool _Intl = false>
class moneypunct;
template<typename _CharT, bool _Intl = false>
class moneypunct_byname;
}
struct messages_base;
namespace __cxx11 {
template<typename _CharT>
class messages;
template<typename _CharT>
class messages_byname;
}
}
#pragma GCC visibility push(default)
namespace __cxxabiv1
{
class __forced_unwind
{
virtual ~__forced_unwind() throw();
virtual void __pure_dummy() = 0;
};
}
#pragma GCC visibility pop
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _CharT, typename _Traits>
inline void
__ostream_write(basic_ostream<_CharT, _Traits>& __out,
const _CharT* __s, streamsize __n)
{
typedef basic_ostream<_CharT, _Traits> __ostream_type;
typedef typename __ostream_type::ios_base __ios_base;
const streamsize __put = __out.rdbuf()->sputn(__s, __n);
if (__put != __n)
__out.setstate(__ios_base::badbit);
}
template<typename _CharT, typename _Traits>
inline void
__ostream_fill(basic_ostream<_CharT, _Traits>& __out, streamsize __n)
{
typedef basic_ostream<_CharT, _Traits> __ostream_type;
typedef typename __ostream_type::ios_base __ios_base;
const _CharT __c = __out.fill();
for (; __n > 0; --__n)
{
const typename _Traits::int_type __put = __out.rdbuf()->sputc(__c);
if (_Traits::eq_int_type(__put, _Traits::eof()))
{
__out.setstate(__ios_base::badbit);
break;
}
}
}
template<typename _CharT, typename _Traits>
basic_ostream<_CharT, _Traits>&
__ostream_insert(basic_ostream<_CharT, _Traits>& __out,
const _CharT* __s, streamsize __n)
{
typedef basic_ostream<_CharT, _Traits> __ostream_type;
typedef typename __ostream_type::ios_base __ios_base;
typename __ostream_type::sentry __cerb(__out);
if (__cerb)
{
try
{
const streamsize __w = __out.width();
if (__w > __n)
{
const bool __left = ((__out.flags()
& __ios_base::adjustfield)
== __ios_base::left);
if (!__left)
__ostream_fill(__out, __w - __n);
if (__out.good())
__ostream_write(__out, __s, __n);
if (__left && __out.good())
__ostream_fill(__out, __w - __n);
}
else
__ostream_write(__out, __s, __n);
__out.width(0);
}
catch(__cxxabiv1::__forced_unwind&)
{
__out._M_setstate(__ios_base::badbit);
throw;
}
catch(...)
{ __out._M_setstate(__ios_base::badbit); }
}
return __out;
}
extern template ostream& __ostream_insert(ostream&, const char*, streamsize);
extern template wostream& __ostream_insert(wostream&, const wchar_t*,
streamsize);
}
namespace std __attribute__ ((__visibility__ ("default")))
{
constexpr size_t
__sv_check(size_t __size, size_t __pos, const char* __s)
{
if (__pos > __size)
__throw_out_of_range_fmt(("%s: __pos (which is %zu) > __size " "(which is %zu)")
, __s, __pos, __size);
return __pos;
}
constexpr size_t
__sv_limit(size_t __size, size_t __pos, size_t __off) noexcept
{
const bool __testoff = __off < __size - __pos;
return __testoff ? __off : __size - __pos;
}
template<typename _CharT, typename _Traits = std::char_traits<_CharT>>
class basic_string_view
{
static_assert(!is_array_v<_CharT>);
static_assert(is_trivial_v<_CharT> && is_standard_layout_v<_CharT>);
static_assert(is_same_v<_CharT, typename _Traits::char_type>);
public:
using traits_type = _Traits;
using value_type = _CharT;
using pointer = value_type*;
using const_pointer = const value_type*;
using reference = value_type&;
using const_reference = const value_type&;
using const_iterator = const value_type*;
using iterator = const_iterator;
using const_reverse_iterator = std::reverse_iterator<const_iterator>;
using reverse_iterator = const_reverse_iterator;
using size_type = size_t;
using difference_type = ptrdiff_t;
static constexpr size_type npos = size_type(-1);
constexpr
basic_string_view() noexcept
: _M_len{0}, _M_str{nullptr}
{ }
constexpr basic_string_view(const basic_string_view&) noexcept = default;
[[__gnu__::__nonnull__]]
constexpr
basic_string_view(const _CharT* __str) noexcept
: _M_len{traits_type::length(__str)},
_M_str{__str}
{ }
constexpr
basic_string_view(const _CharT* __str, size_type __len) noexcept
: _M_len{__len}, _M_str{__str}
{ }
constexpr basic_string_view&
operator=(const basic_string_view&) noexcept = default;
[[nodiscard]]
constexpr const_iterator
begin() const noexcept
{ return this->_M_str; }
[[nodiscard]]
constexpr const_iterator
end() const noexcept
{ return this->_M_str + this->_M_len; }
[[nodiscard]]
constexpr const_iterator
cbegin() const noexcept
{ return this->_M_str; }
[[nodiscard]]
constexpr const_iterator
cend() const noexcept
{ return this->_M_str + this->_M_len; }
[[nodiscard]]
constexpr const_reverse_iterator
rbegin() const noexcept
{ return const_reverse_iterator(this->end()); }
[[nodiscard]]
constexpr const_reverse_iterator
rend() const noexcept
{ return const_reverse_iterator(this->begin()); }
[[nodiscard]]
constexpr const_reverse_iterator
crbegin() const noexcept
{ return const_reverse_iterator(this->end()); }
[[nodiscard]]
constexpr const_reverse_iterator
crend() const noexcept
{ return const_reverse_iterator(this->begin()); }
[[nodiscard]]
constexpr size_type
size() const noexcept
{ return this->_M_len; }
[[nodiscard]]
constexpr size_type
length() const noexcept
{ return _M_len; }
[[nodiscard]]
constexpr size_type
max_size() const noexcept
{
return (npos - sizeof(size_type) - sizeof(void*))
/ sizeof(value_type) / 4;
}
[[nodiscard]]
constexpr bool
empty() const noexcept
{ return this->_M_len == 0; }
[[nodiscard]]
constexpr const_reference
operator[](size_type __pos) const noexcept
{
do { if (std::__is_constant_evaluated() && !bool(__pos < this->_M_len)) __builtin_unreachable(); } while (false);
return *(this->_M_str + __pos);
}
[[nodiscard]]
constexpr const_reference
at(size_type __pos) const
{
if (__pos >= _M_len)
__throw_out_of_range_fmt(("basic_string_view::at: __pos " "(which is %zu) >= this->size() " "(which is %zu)")
, __pos, this->size());
return *(this->_M_str + __pos);
}
[[nodiscard]]
constexpr const_reference
front() const noexcept
{
do { if (std::__is_constant_evaluated() && !bool(this->_M_len > 0)) __builtin_unreachable(); } while (false);
return *this->_M_str;
}
[[nodiscard]]
constexpr const_reference
back() const noexcept
{
do { if (std::__is_constant_evaluated() && !bool(this->_M_len > 0)) __builtin_unreachable(); } while (false);
return *(this->_M_str + this->_M_len - 1);
}
[[nodiscard]]
constexpr const_pointer
data() const noexcept
{ return this->_M_str; }
constexpr void
remove_prefix(size_type __n) noexcept
{
do { if (std::__is_constant_evaluated() && !bool(this->_M_len >= __n)) __builtin_unreachable(); } while (false);
this->_M_str += __n;
this->_M_len -= __n;
}
constexpr void
remove_suffix(size_type __n) noexcept
{ this->_M_len -= __n; }
constexpr void
swap(basic_string_view& __sv) noexcept
{
auto __tmp = *this;
*this = __sv;
__sv = __tmp;
}
size_type
copy(_CharT* __str, size_type __n, size_type __pos = 0) const
{
;
__pos = std::__sv_check(size(), __pos, "basic_string_view::copy");
const size_type __rlen = std::min<size_t>(__n, _M_len - __pos);
traits_type::copy(__str, data() + __pos, __rlen);
return __rlen;
}
[[nodiscard]]
constexpr basic_string_view
substr(size_type __pos = 0, size_type __n = npos) const noexcept(false)
{
__pos = std::__sv_check(size(), __pos, "basic_string_view::substr");
const size_type __rlen = std::min<size_t>(__n, _M_len - __pos);
return basic_string_view{_M_str + __pos, __rlen};
}
[[nodiscard]]
constexpr int
compare(basic_string_view __str) const noexcept
{
const size_type __rlen = std::min(this->_M_len, __str._M_len);
int __ret = traits_type::compare(this->_M_str, __str._M_str, __rlen);
if (__ret == 0)
__ret = _S_compare(this->_M_len, __str._M_len);
return __ret;
}
[[nodiscard]]
constexpr int
compare(size_type __pos1, size_type __n1, basic_string_view __str) const
{ return this->substr(__pos1, __n1).compare(__str); }
[[nodiscard]]
constexpr int
compare(size_type __pos1, size_type __n1,
basic_string_view __str, size_type __pos2, size_type __n2) const
{
return this->substr(__pos1, __n1).compare(__str.substr(__pos2, __n2));
}
[[nodiscard, __gnu__::__nonnull__]]
constexpr int
compare(const _CharT* __str) const noexcept
{ return this->compare(basic_string_view{__str}); }
[[nodiscard, __gnu__::__nonnull__]]
constexpr int
compare(size_type __pos1, size_type __n1, const _CharT* __str) const
{ return this->substr(__pos1, __n1).compare(basic_string_view{__str}); }
[[nodiscard]]
constexpr int
compare(size_type __pos1, size_type __n1,
const _CharT* __str, size_type __n2) const noexcept(false)
{
return this->substr(__pos1, __n1)
.compare(basic_string_view(__str, __n2));
}
[[nodiscard]]
constexpr size_type
find(basic_string_view __str, size_type __pos = 0) const noexcept
{ return this->find(__str._M_str, __pos, __str._M_len); }
[[nodiscard]]
constexpr size_type
find(_CharT __c, size_type __pos = 0) const noexcept;
[[nodiscard]]
constexpr size_type
find(const _CharT* __str, size_type __pos, size_type __n) const noexcept;
[[nodiscard, __gnu__::__nonnull__]]
constexpr size_type
find(const _CharT* __str, size_type __pos = 0) const noexcept
{ return this->find(__str, __pos, traits_type::length(__str)); }
[[nodiscard]]
constexpr size_type
rfind(basic_string_view __str, size_type __pos = npos) const noexcept
{ return this->rfind(__str._M_str, __pos, __str._M_len); }
[[nodiscard]]
constexpr size_type
rfind(_CharT __c, size_type __pos = npos) const noexcept;
[[nodiscard]]
constexpr size_type
rfind(const _CharT* __str, size_type __pos, size_type __n) const noexcept;
[[nodiscard, __gnu__::__nonnull__]]
constexpr size_type
rfind(const _CharT* __str, size_type __pos = npos) const noexcept
{ return this->rfind(__str, __pos, traits_type::length(__str)); }
[[nodiscard]]
constexpr size_type
find_first_of(basic_string_view __str, size_type __pos = 0) const noexcept
{ return this->find_first_of(__str._M_str, __pos, __str._M_len); }
[[nodiscard]]
constexpr size_type
find_first_of(_CharT __c, size_type __pos = 0) const noexcept
{ return this->find(__c, __pos); }
[[nodiscard]]
constexpr size_type
find_first_of(const _CharT* __str, size_type __pos,
size_type __n) const noexcept;
[[nodiscard, __gnu__::__nonnull__]]
constexpr size_type
find_first_of(const _CharT* __str, size_type __pos = 0) const noexcept
{ return this->find_first_of(__str, __pos, traits_type::length(__str)); }
[[nodiscard]]
constexpr size_type
find_last_of(basic_string_view __str,
size_type __pos = npos) const noexcept
{ return this->find_last_of(__str._M_str, __pos, __str._M_len); }
[[nodiscard]]
constexpr size_type
find_last_of(_CharT __c, size_type __pos=npos) const noexcept
{ return this->rfind(__c, __pos); }
[[nodiscard]]
constexpr size_type
find_last_of(const _CharT* __str, size_type __pos,
size_type __n) const noexcept;
[[nodiscard, __gnu__::__nonnull__]]
constexpr size_type
find_last_of(const _CharT* __str, size_type __pos = npos) const noexcept
{ return this->find_last_of(__str, __pos, traits_type::length(__str)); }
[[nodiscard]]
constexpr size_type
find_first_not_of(basic_string_view __str,
size_type __pos = 0) const noexcept
{ return this->find_first_not_of(__str._M_str, __pos, __str._M_len); }
[[nodiscard]]
constexpr size_type
find_first_not_of(_CharT __c, size_type __pos = 0) const noexcept;
[[nodiscard]]
constexpr size_type
find_first_not_of(const _CharT* __str,
size_type __pos, size_type __n) const noexcept;
[[nodiscard, __gnu__::__nonnull__]]
constexpr size_type
find_first_not_of(const _CharT* __str, size_type __pos = 0) const noexcept
{
return this->find_first_not_of(__str, __pos,
traits_type::length(__str));
}
[[nodiscard]]
constexpr size_type
find_last_not_of(basic_string_view __str,
size_type __pos = npos) const noexcept
{ return this->find_last_not_of(__str._M_str, __pos, __str._M_len); }
[[nodiscard]]
constexpr size_type
find_last_not_of(_CharT __c, size_type __pos = npos) const noexcept;
[[nodiscard]]
constexpr size_type
find_last_not_of(const _CharT* __str,
size_type __pos, size_type __n) const noexcept;
[[nodiscard, __gnu__::__nonnull__]]
constexpr size_type
find_last_not_of(const _CharT* __str,
size_type __pos = npos) const noexcept
{
return this->find_last_not_of(__str, __pos,
traits_type::length(__str));
}
private:
static constexpr int
_S_compare(size_type __n1, size_type __n2) noexcept
{
using __limits = __gnu_cxx::__int_traits<int>;
const difference_type __diff = __n1 - __n2;
if (__diff > __limits::__max)
return __limits::__max;
if (__diff < __limits::__min)
return __limits::__min;
return static_cast<int>(__diff);
}
size_t _M_len;
const _CharT* _M_str;
};
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator==(basic_string_view<_CharT, _Traits> __x,
basic_string_view<_CharT, _Traits> __y) noexcept
{ return __x.size() == __y.size() && __x.compare(__y) == 0; }
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator==(basic_string_view<_CharT, _Traits> __x,
__type_identity_t<basic_string_view<_CharT, _Traits>> __y)
noexcept
{ return __x.size() == __y.size() && __x.compare(__y) == 0; }
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator==(__type_identity_t<basic_string_view<_CharT, _Traits>> __x,
basic_string_view<_CharT, _Traits> __y) noexcept
{ return __x.size() == __y.size() && __x.compare(__y) == 0; }
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator!=(basic_string_view<_CharT, _Traits> __x,
basic_string_view<_CharT, _Traits> __y) noexcept
{ return !(__x == __y); }
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator!=(basic_string_view<_CharT, _Traits> __x,
__type_identity_t<basic_string_view<_CharT, _Traits>> __y)
noexcept
{ return !(__x == __y); }
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator!=(__type_identity_t<basic_string_view<_CharT, _Traits>> __x,
basic_string_view<_CharT, _Traits> __y) noexcept
{ return !(__x == __y); }
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator< (basic_string_view<_CharT, _Traits> __x,
basic_string_view<_CharT, _Traits> __y) noexcept
{ return __x.compare(__y) < 0; }
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator< (basic_string_view<_CharT, _Traits> __x,
__type_identity_t<basic_string_view<_CharT, _Traits>> __y)
noexcept
{ return __x.compare(__y) < 0; }
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator< (__type_identity_t<basic_string_view<_CharT, _Traits>> __x,
basic_string_view<_CharT, _Traits> __y) noexcept
{ return __x.compare(__y) < 0; }
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator> (basic_string_view<_CharT, _Traits> __x,
basic_string_view<_CharT, _Traits> __y) noexcept
{ return __x.compare(__y) > 0; }
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator> (basic_string_view<_CharT, _Traits> __x,
__type_identity_t<basic_string_view<_CharT, _Traits>> __y)
noexcept
{ return __x.compare(__y) > 0; }
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator> (__type_identity_t<basic_string_view<_CharT, _Traits>> __x,
basic_string_view<_CharT, _Traits> __y) noexcept
{ return __x.compare(__y) > 0; }
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator<=(basic_string_view<_CharT, _Traits> __x,
basic_string_view<_CharT, _Traits> __y) noexcept
{ return __x.compare(__y) <= 0; }
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator<=(basic_string_view<_CharT, _Traits> __x,
__type_identity_t<basic_string_view<_CharT, _Traits>> __y)
noexcept
{ return __x.compare(__y) <= 0; }
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator<=(__type_identity_t<basic_string_view<_CharT, _Traits>> __x,
basic_string_view<_CharT, _Traits> __y) noexcept
{ return __x.compare(__y) <= 0; }
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator>=(basic_string_view<_CharT, _Traits> __x,
basic_string_view<_CharT, _Traits> __y) noexcept
{ return __x.compare(__y) >= 0; }
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator>=(basic_string_view<_CharT, _Traits> __x,
__type_identity_t<basic_string_view<_CharT, _Traits>> __y)
noexcept
{ return __x.compare(__y) >= 0; }
template<typename _CharT, typename _Traits>
[[nodiscard]]
constexpr bool
operator>=(__type_identity_t<basic_string_view<_CharT, _Traits>> __x,
basic_string_view<_CharT, _Traits> __y) noexcept
{ return __x.compare(__y) >= 0; }
template<typename _CharT, typename _Traits>
inline basic_ostream<_CharT, _Traits>&
operator<<(basic_ostream<_CharT, _Traits>& __os,
basic_string_view<_CharT,_Traits> __str)
{ return __ostream_insert(__os, __str.data(), __str.size()); }
using string_view = basic_string_view<char>;
using wstring_view = basic_string_view<wchar_t>;
using u16string_view = basic_string_view<char16_t>;
using u32string_view = basic_string_view<char32_t>;
template<typename _Tp>
struct hash;
template<>
struct hash<string_view>
: public __hash_base<size_t, string_view>
{
[[nodiscard]]
size_t
operator()(const string_view& __str) const noexcept
{ return std::_Hash_impl::hash(__str.data(), __str.length()); }
};
template<>
struct __is_fast_hash<hash<string_view>> : std::false_type
{ };
template<>
struct hash<wstring_view>
: public __hash_base<size_t, wstring_view>
{
[[nodiscard]]
size_t
operator()(const wstring_view& __s) const noexcept
{ return std::_Hash_impl::hash(__s.data(),
__s.length() * sizeof(wchar_t)); }
};
template<>
struct __is_fast_hash<hash<wstring_view>> : std::false_type
{ };
template<>
struct hash<u16string_view>
: public __hash_base<size_t, u16string_view>
{
[[nodiscard]]
size_t
operator()(const u16string_view& __s) const noexcept
{ return std::_Hash_impl::hash(__s.data(),
__s.length() * sizeof(char16_t)); }
};
template<>
struct __is_fast_hash<hash<u16string_view>> : std::false_type
{ };
template<>
struct hash<u32string_view>
: public __hash_base<size_t, u32string_view>
{
[[nodiscard]]
size_t
operator()(const u32string_view& __s) const noexcept
{ return std::_Hash_impl::hash(__s.data(),
__s.length() * sizeof(char32_t)); }
};
template<>
struct __is_fast_hash<hash<u32string_view>> : std::false_type
{ };
inline namespace literals
{
inline namespace string_view_literals
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wliteral-suffix"
inline constexpr basic_string_view<char>
operator""sv(const char* __str, size_t __len) noexcept
{ return basic_string_view<char>{__str, __len}; }
inline constexpr basic_string_view<wchar_t>
operator""sv(const wchar_t* __str, size_t __len) noexcept
{ return basic_string_view<wchar_t>{__str, __len}; }
inline constexpr basic_string_view<char16_t>
operator""sv(const char16_t* __str, size_t __len) noexcept
{ return basic_string_view<char16_t>{__str, __len}; }
inline constexpr basic_string_view<char32_t>
operator""sv(const char32_t* __str, size_t __len) noexcept
{ return basic_string_view<char32_t>{__str, __len}; }
#pragma GCC diagnostic pop
}
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _CharT, typename _Traits>
constexpr typename basic_string_view<_CharT, _Traits>::size_type
basic_string_view<_CharT, _Traits>::
find(const _CharT* __str, size_type __pos, size_type __n) const noexcept
{
;
if (__n == 0)
return __pos <= _M_len ? __pos : npos;
if (__pos >= _M_len)
return npos;
const _CharT __elem0 = __str[0];
const _CharT* __first = _M_str + __pos;
const _CharT* const __last = _M_str + _M_len;
size_type __len = _M_len - __pos;
while (__len >= __n)
{
__first = traits_type::find(__first, __len - __n + 1, __elem0);
if (!__first)
return npos;
if (traits_type::compare(__first, __str, __n) == 0)
return __first - _M_str;
__len = __last - ++__first;
}
return npos;
}
template<typename _CharT, typename _Traits>
constexpr typename basic_string_view<_CharT, _Traits>::size_type
basic_string_view<_CharT, _Traits>::
find(_CharT __c, size_type __pos) const noexcept
{
size_type __ret = npos;
if (__pos < this->_M_len)
{
const size_type __n = this->_M_len - __pos;
const _CharT* __p = traits_type::find(this->_M_str + __pos, __n, __c);
if (__p)
__ret = __p - this->_M_str;
}
return __ret;
}
template<typename _CharT, typename _Traits>
constexpr typename basic_string_view<_CharT, _Traits>::size_type
basic_string_view<_CharT, _Traits>::
rfind(const _CharT* __str, size_type __pos, size_type __n) const noexcept
{
;
if (__n <= this->_M_len)
{
__pos = std::min(size_type(this->_M_len - __n), __pos);
do
{
if (traits_type::compare(this->_M_str + __pos, __str, __n) == 0)
return __pos;
}
while (__pos-- > 0);
}
return npos;
}
template<typename _CharT, typename _Traits>
constexpr typename basic_string_view<_CharT, _Traits>::size_type
basic_string_view<_CharT, _Traits>::
rfind(_CharT __c, size_type __pos) const noexcept
{
size_type __size = this->_M_len;
if (__size > 0)
{
if (--__size > __pos)
__size = __pos;
for (++__size; __size-- > 0; )
if (traits_type::eq(this->_M_str[__size], __c))
return __size;
}
return npos;
}
template<typename _CharT, typename _Traits>
constexpr typename basic_string_view<_CharT, _Traits>::size_type
basic_string_view<_CharT, _Traits>::
find_first_of(const _CharT* __str, size_type __pos,
size_type __n) const noexcept
{
;
for (; __n && __pos < this->_M_len; ++__pos)
{
const _CharT* __p = traits_type::find(__str, __n,
this->_M_str[__pos]);
if (__p)
return __pos;
}
return npos;
}
template<typename _CharT, typename _Traits>
constexpr typename basic_string_view<_CharT, _Traits>::size_type
basic_string_view<_CharT, _Traits>::
find_last_of(const _CharT* __str, size_type __pos,
size_type __n) const noexcept
{
;
size_type __size = this->size();
if (__size && __n)
{
if (--__size > __pos)
__size = __pos;
do
{
if (traits_type::find(__str, __n, this->_M_str[__size]))
return __size;
}
while (__size-- != 0);
}
return npos;
}
template<typename _CharT, typename _Traits>
constexpr typename basic_string_view<_CharT, _Traits>::size_type
basic_string_view<_CharT, _Traits>::
find_first_not_of(const _CharT* __str, size_type __pos,
size_type __n) const noexcept
{
;
for (; __pos < this->_M_len; ++__pos)
if (!traits_type::find(__str, __n, this->_M_str[__pos]))
return __pos;
return npos;
}
template<typename _CharT, typename _Traits>
constexpr typename basic_string_view<_CharT, _Traits>::size_type
basic_string_view<_CharT, _Traits>::
find_first_not_of(_CharT __c, size_type __pos) const noexcept
{
for (; __pos < this->_M_len; ++__pos)
if (!traits_type::eq(this->_M_str[__pos], __c))
return __pos;
return npos;
}
template<typename _CharT, typename _Traits>
constexpr typename basic_string_view<_CharT, _Traits>::size_type
basic_string_view<_CharT, _Traits>::
find_last_not_of(const _CharT* __str, size_type __pos,
size_type __n) const noexcept
{
;
size_type __size = this->_M_len;
if (__size)
{
if (--__size > __pos)
__size = __pos;
do
{
if (!traits_type::find(__str, __n, this->_M_str[__size]))
return __size;
}
while (__size--);
}
return npos;
}
template<typename _CharT, typename _Traits>
constexpr typename basic_string_view<_CharT, _Traits>::size_type
basic_string_view<_CharT, _Traits>::
find_last_not_of(_CharT __c, size_type __pos) const noexcept
{
size_type __size = this->_M_len;
if (__size)
{
if (--__size > __pos)
__size = __pos;
do
{
if (!traits_type::eq(this->_M_str[__size], __c))
return __size;
}
while (__size--);
}
return npos;
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __cxx11 {
template<typename _CharT, typename _Traits, typename _Alloc>
class basic_string
{
typedef typename __gnu_cxx::__alloc_traits<_Alloc>::template
rebind<_CharT>::other _Char_alloc_type;
typedef __gnu_cxx::__alloc_traits<_Char_alloc_type> _Alloc_traits;
public:
typedef _Traits traits_type;
typedef typename _Traits::char_type value_type;
typedef _Char_alloc_type allocator_type;
typedef typename _Alloc_traits::size_type size_type;
typedef typename _Alloc_traits::difference_type difference_type;
typedef typename _Alloc_traits::reference reference;
typedef typename _Alloc_traits::const_reference const_reference;
typedef typename _Alloc_traits::pointer pointer;
typedef typename _Alloc_traits::const_pointer const_pointer;
typedef __gnu_cxx::__normal_iterator<pointer, basic_string> iterator;
typedef __gnu_cxx::__normal_iterator<const_pointer, basic_string>
const_iterator;
typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
typedef std::reverse_iterator<iterator> reverse_iterator;
static const size_type npos = static_cast<size_type>(-1);
protected:
typedef const_iterator __const_iterator;
private:
static pointer
_S_allocate(_Char_alloc_type& __a, size_type __n)
{
pointer __p = _Alloc_traits::allocate(__a, __n);
return __p;
}
typedef basic_string_view<_CharT, _Traits> __sv_type;
template<typename _Tp, typename _Res>
using _If_sv = enable_if_t<
__and_<is_convertible<const _Tp&, __sv_type>,
__not_<is_convertible<const _Tp*, const basic_string*>>,
__not_<is_convertible<const _Tp&, const _CharT*>>>::value,
_Res>;
static __sv_type
_S_to_string_view(__sv_type __svt) noexcept
{ return __svt; }
struct __sv_wrapper
{
explicit
__sv_wrapper(__sv_type __sv) noexcept : _M_sv(__sv) { }
__sv_type _M_sv;
};
explicit
basic_string(__sv_wrapper __svw, const _Alloc& __a)
: basic_string(__svw._M_sv.data(), __svw._M_sv.size(), __a) { }
struct _Alloc_hider : allocator_type
{
_Alloc_hider(pointer __dat, const _Alloc& __a)
: allocator_type(__a), _M_p(__dat) { }
_Alloc_hider(pointer __dat, _Alloc&& __a = _Alloc())
: allocator_type(std::move(__a)), _M_p(__dat) { }
pointer _M_p;
};
_Alloc_hider _M_dataplus;
size_type _M_string_length;
enum { _S_local_capacity = 15 / sizeof(_CharT) };
union
{
_CharT _M_local_buf[_S_local_capacity + 1];
size_type _M_allocated_capacity;
};
void
_M_data(pointer __p)
{ _M_dataplus._M_p = __p; }
void
_M_length(size_type __length)
{ _M_string_length = __length; }
pointer
_M_data() const
{ return _M_dataplus._M_p; }
pointer
_M_local_data()
{
return std::pointer_traits<pointer>::pointer_to(*_M_local_buf);
}
const_pointer
_M_local_data() const
{
return std::pointer_traits<const_pointer>::pointer_to(*_M_local_buf);
}
void
_M_capacity(size_type __capacity)
{ _M_allocated_capacity = __capacity; }
void
_M_set_length(size_type __n)
{
_M_length(__n);
traits_type::assign(_M_data()[__n], _CharT());
}
bool
_M_is_local() const
{
if (_M_data() == _M_local_data())
{
if (_M_string_length > _S_local_capacity)
__builtin_unreachable();
return true;
}
return false;
}
pointer
_M_create(size_type&, size_type);
void
_M_dispose()
{
if (!_M_is_local())
_M_destroy(_M_allocated_capacity);
}
void
_M_destroy(size_type __size) throw()
{ _Alloc_traits::deallocate(_M_get_allocator(), _M_data(), __size + 1); }
template<typename _InIterator>
void
_M_construct(_InIterator __beg, _InIterator __end,
std::input_iterator_tag);
template<typename _FwdIterator>
void
_M_construct(_FwdIterator __beg, _FwdIterator __end,
std::forward_iterator_tag);
void
_M_construct(size_type __req, _CharT __c);
allocator_type&
_M_get_allocator()
{ return _M_dataplus; }
const allocator_type&
_M_get_allocator() const
{ return _M_dataplus; }
__attribute__((__always_inline__))
constexpr
pointer
_M_use_local_data() noexcept
{
return _M_local_data();
}
private:
size_type
_M_check(size_type __pos, const char* __s) const
{
if (__pos > this->size())
__throw_out_of_range_fmt(("%s: __pos (which is %zu) > " "this->size() (which is %zu)")
,
__s, __pos, this->size());
return __pos;
}
void
_M_check_length(size_type __n1, size_type __n2, const char* __s) const
{
if (this->max_size() - (this->size() - __n1) < __n2)
__throw_length_error((__s));
}
size_type
_M_limit(size_type __pos, size_type __off) const noexcept
{
const bool __testoff = __off < this->size() - __pos;
return __testoff ? __off : this->size() - __pos;
}
bool
_M_disjunct(const _CharT* __s) const noexcept
{
return (less<const _CharT*>()(__s, _M_data())
|| less<const _CharT*>()(_M_data() + this->size(), __s));
}
static void
_S_copy(_CharT* __d, const _CharT* __s, size_type __n)
{
if (__n == 1)
traits_type::assign(*__d, *__s);
else
traits_type::copy(__d, __s, __n);
}
static void
_S_move(_CharT* __d, const _CharT* __s, size_type __n)
{
if (__n == 1)
traits_type::assign(*__d, *__s);
else
traits_type::move(__d, __s, __n);
}
static void
_S_assign(_CharT* __d, size_type __n, _CharT __c)
{
if (__n == 1)
traits_type::assign(*__d, __c);
else
traits_type::assign(__d, __n, __c);
}
template<class _Iterator>
static void
_S_copy_chars(_CharT* __p, _Iterator __k1, _Iterator __k2)
{
for (; __k1 != __k2; ++__k1, (void)++__p)
traits_type::assign(*__p, *__k1);
}
static void
_S_copy_chars(_CharT* __p, iterator __k1, iterator __k2) noexcept
{ _S_copy_chars(__p, __k1.base(), __k2.base()); }
static void
_S_copy_chars(_CharT* __p, const_iterator __k1, const_iterator __k2)
noexcept
{ _S_copy_chars(__p, __k1.base(), __k2.base()); }
static void
_S_copy_chars(_CharT* __p, _CharT* __k1, _CharT* __k2) noexcept
{ _S_copy(__p, __k1, __k2 - __k1); }
static void
_S_copy_chars(_CharT* __p, const _CharT* __k1, const _CharT* __k2)
noexcept
{ _S_copy(__p, __k1, __k2 - __k1); }
static int
_S_compare(size_type __n1, size_type __n2) noexcept
{
const difference_type __d = difference_type(__n1 - __n2);
if (__d > __gnu_cxx::__numeric_traits<int>::__max)
return __gnu_cxx::__numeric_traits<int>::__max;
else if (__d < __gnu_cxx::__numeric_traits<int>::__min)
return __gnu_cxx::__numeric_traits<int>::__min;
else
return int(__d);
}
void
_M_assign(const basic_string&);
void
_M_mutate(size_type __pos, size_type __len1, const _CharT* __s,
size_type __len2);
void
_M_erase(size_type __pos, size_type __n);
public:
basic_string()
noexcept(is_nothrow_default_constructible<_Alloc>::value)
: _M_dataplus(_M_local_data())
{
_M_use_local_data();
_M_set_length(0);
}
explicit
basic_string(const _Alloc& __a) noexcept
: _M_dataplus(_M_local_data(), __a)
{
_M_use_local_data();
_M_set_length(0);
}
basic_string(const basic_string& __str)
: _M_dataplus(_M_local_data(),
_Alloc_traits::_S_select_on_copy(__str._M_get_allocator()))
{
_M_construct(__str._M_data(), __str._M_data() + __str.length(),
std::forward_iterator_tag());
}
basic_string(const basic_string& __str, size_type __pos,
const _Alloc& __a = _Alloc())
: _M_dataplus(_M_local_data(), __a)
{
const _CharT* __start = __str._M_data()
+ __str._M_check(__pos, "basic_string::basic_string");
_M_construct(__start, __start + __str._M_limit(__pos, npos),
std::forward_iterator_tag());
}
basic_string(const basic_string& __str, size_type __pos,
size_type __n)
: _M_dataplus(_M_local_data())
{
const _CharT* __start = __str._M_data()
+ __str._M_check(__pos, "basic_string::basic_string");
_M_construct(__start, __start + __str._M_limit(__pos, __n),
std::forward_iterator_tag());
}
basic_string(const basic_string& __str, size_type __pos,
size_type __n, const _Alloc& __a)
: _M_dataplus(_M_local_data(), __a)
{
const _CharT* __start
= __str._M_data() + __str._M_check(__pos, "string::string");
_M_construct(__start, __start + __str._M_limit(__pos, __n),
std::forward_iterator_tag());
}
basic_string(const _CharT* __s, size_type __n,
const _Alloc& __a = _Alloc())
: _M_dataplus(_M_local_data(), __a)
{
if (__s == 0 && __n > 0)
std::__throw_logic_error(("basic_string: " "construction from null is not valid")
);
_M_construct(__s, __s + __n, std::forward_iterator_tag());
}
template<typename = _RequireAllocator<_Alloc>>
basic_string(const _CharT* __s, const _Alloc& __a = _Alloc())
: _M_dataplus(_M_local_data(), __a)
{
if (__s == 0)
std::__throw_logic_error(("basic_string: " "construction from null is not valid")
);
const _CharT* __end = __s + traits_type::length(__s);
_M_construct(__s, __end, forward_iterator_tag());
}
template<typename = _RequireAllocator<_Alloc>>
basic_string(size_type __n, _CharT __c, const _Alloc& __a = _Alloc())
: _M_dataplus(_M_local_data(), __a)
{ _M_construct(__n, __c); }
basic_string(basic_string&& __str) noexcept
: _M_dataplus(_M_local_data(), std::move(__str._M_get_allocator()))
{
if (__str._M_is_local())
{
traits_type::copy(_M_local_buf, __str._M_local_buf,
__str.length() + 1);
}
else
{
_M_data(__str._M_data());
_M_capacity(__str._M_allocated_capacity);
}
_M_length(__str.length());
__str._M_data(__str._M_local_data());
__str._M_set_length(0);
}
basic_string(initializer_list<_CharT> __l, const _Alloc& __a = _Alloc())
: _M_dataplus(_M_local_data(), __a)
{ _M_construct(__l.begin(), __l.end(), std::forward_iterator_tag()); }
basic_string(const basic_string& __str, const _Alloc& __a)
: _M_dataplus(_M_local_data(), __a)
{ _M_construct(__str.begin(), __str.end(), std::forward_iterator_tag()); }
basic_string(basic_string&& __str, const _Alloc& __a)
noexcept(_Alloc_traits::_S_always_equal())
: _M_dataplus(_M_local_data(), __a)
{
if (__str._M_is_local())
{
traits_type::copy(_M_local_buf, __str._M_local_buf,
__str.length() + 1);
_M_length(__str.length());
__str._M_set_length(0);
}
else if (_Alloc_traits::_S_always_equal()
|| __str.get_allocator() == __a)
{
_M_data(__str._M_data());
_M_length(__str.length());
_M_capacity(__str._M_allocated_capacity);
__str._M_data(__str._M_local_buf);
__str._M_set_length(0);
}
else
_M_construct(__str.begin(), __str.end(), std::forward_iterator_tag());
}
template<typename _InputIterator,
typename = std::_RequireInputIter<_InputIterator>>
basic_string(_InputIterator __beg, _InputIterator __end,
const _Alloc& __a = _Alloc())
: _M_dataplus(_M_local_data(), __a), _M_string_length(0)
{
_M_construct(__beg, __end, std::__iterator_category(__beg));
}
template<typename _Tp,
typename = enable_if_t<is_convertible_v<const _Tp&, __sv_type>>>
basic_string(const _Tp& __t, size_type __pos, size_type __n,
const _Alloc& __a = _Alloc())
: basic_string(_S_to_string_view(__t).substr(__pos, __n), __a) { }
template<typename _Tp, typename = _If_sv<_Tp, void>>
explicit
basic_string(const _Tp& __t, const _Alloc& __a = _Alloc())
: basic_string(__sv_wrapper(_S_to_string_view(__t)), __a) { }
~basic_string()
{ _M_dispose(); }
basic_string&
operator=(const basic_string& __str)
{
return this->assign(__str);
}
basic_string&
operator=(const _CharT* __s)
{ return this->assign(__s); }
basic_string&
operator=(_CharT __c)
{
this->assign(1, __c);
return *this;
}
basic_string&
operator=(basic_string&& __str)
noexcept(_Alloc_traits::_S_nothrow_move())
{
const bool __equal_allocs = _Alloc_traits::_S_always_equal()
|| _M_get_allocator() == __str._M_get_allocator();
if (!_M_is_local() && _Alloc_traits::_S_propagate_on_move_assign()
&& !__equal_allocs)
{
_M_destroy(_M_allocated_capacity);
_M_data(_M_local_data());
_M_set_length(0);
}
std::__alloc_on_move(_M_get_allocator(), __str._M_get_allocator());
if (__str._M_is_local())
{
if (__builtin_expect(std::__addressof(__str) != this, true))
{
if (__str.size())
this->_S_copy(_M_data(), __str._M_data(), __str.size());
_M_set_length(__str.size());
}
}
else if (_Alloc_traits::_S_propagate_on_move_assign() || __equal_allocs)
{
pointer __data = nullptr;
size_type __capacity;
if (!_M_is_local())
{
if (__equal_allocs)
{
__data = _M_data();
__capacity = _M_allocated_capacity;
}
else
_M_destroy(_M_allocated_capacity);
}
_M_data(__str._M_data());
_M_length(__str.length());
_M_capacity(__str._M_allocated_capacity);
if (__data)
{
__str._M_data(__data);
__str._M_capacity(__capacity);
}
else
__str._M_data(__str._M_local_buf);
}
else
assign(__str);
__str.clear();
return *this;
}
basic_string&
operator=(initializer_list<_CharT> __l)
{
this->assign(__l.begin(), __l.size());
return *this;
}
template<typename _Tp>
_If_sv<_Tp, basic_string&>
operator=(const _Tp& __svt)
{ return this->assign(__svt); }
operator __sv_type() const noexcept
{ return __sv_type(data(), size()); }
[[__nodiscard__]]
iterator
begin() noexcept
{ return iterator(_M_data()); }
[[__nodiscard__]]
const_iterator
begin() const noexcept
{ return const_iterator(_M_data()); }
[[__nodiscard__]]
iterator
end() noexcept
{ return iterator(_M_data() + this->size()); }
[[__nodiscard__]]
const_iterator
end() const noexcept
{ return const_iterator(_M_data() + this->size()); }
[[__nodiscard__]]
reverse_iterator
rbegin() noexcept
{ return reverse_iterator(this->end()); }
[[__nodiscard__]]
const_reverse_iterator
rbegin() const noexcept
{ return const_reverse_iterator(this->end()); }
[[__nodiscard__]]
reverse_iterator
rend() noexcept
{ return reverse_iterator(this->begin()); }
[[__nodiscard__]]
const_reverse_iterator
rend() const noexcept
{ return const_reverse_iterator(this->begin()); }
[[__nodiscard__]]
const_iterator
cbegin() const noexcept
{ return const_iterator(this->_M_data()); }
[[__nodiscard__]]
const_iterator
cend() const noexcept
{ return const_iterator(this->_M_data() + this->size()); }
[[__nodiscard__]]
const_reverse_iterator
crbegin() const noexcept
{ return const_reverse_iterator(this->end()); }
[[__nodiscard__]]
const_reverse_iterator
crend() const noexcept
{ return const_reverse_iterator(this->begin()); }
public:
[[__nodiscard__]]
size_type
size() const noexcept
{ return _M_string_length; }
[[__nodiscard__]]
size_type
length() const noexcept
{ return _M_string_length; }
[[__nodiscard__]]
size_type
max_size() const noexcept
{ return (_Alloc_traits::max_size(_M_get_allocator()) - 1) / 2; }
void
resize(size_type __n, _CharT __c);
void
resize(size_type __n)
{ this->resize(__n, _CharT()); }
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
void
shrink_to_fit() noexcept
{ reserve(); }
#pragma GCC diagnostic pop
[[__nodiscard__]]
size_type
capacity() const noexcept
{
return _M_is_local() ? size_type(_S_local_capacity)
: _M_allocated_capacity;
}
void
reserve(size_type __res_arg);
void
reserve();
void
clear() noexcept
{ _M_set_length(0); }
[[__nodiscard__]]
bool
empty() const noexcept
{ return this->size() == 0; }
[[__nodiscard__]]
const_reference
operator[] (size_type __pos) const noexcept
{
do { if (std::__is_constant_evaluated() && !bool(__pos <= size())) __builtin_unreachable(); } while (false);
return _M_data()[__pos];
}
[[__nodiscard__]]
reference
operator[](size_type __pos)
{
do { if (std::__is_constant_evaluated() && !bool(__pos <= size())) __builtin_unreachable(); } while (false);
;
return _M_data()[__pos];
}
[[__nodiscard__]]
const_reference
at(size_type __n) const
{
if (__n >= this->size())
__throw_out_of_range_fmt(("basic_string::at: __n " "(which is %zu) >= this->size() " "(which is %zu)")
,
__n, this->size());
return _M_data()[__n];
}
[[__nodiscard__]]
reference
at(size_type __n)
{
if (__n >= size())
__throw_out_of_range_fmt(("basic_string::at: __n " "(which is %zu) >= this->size() " "(which is %zu)")
,
__n, this->size());
return _M_data()[__n];
}
[[__nodiscard__]]
reference
front() noexcept
{
do { if (std::__is_constant_evaluated() && !bool(!empty())) __builtin_unreachable(); } while (false);
return operator[](0);
}
[[__nodiscard__]]
const_reference
front() const noexcept
{
do { if (std::__is_constant_evaluated() && !bool(!empty())) __builtin_unreachable(); } while (false);
return operator[](0);
}
[[__nodiscard__]]
reference
back() noexcept
{
do { if (std::__is_constant_evaluated() && !bool(!empty())) __builtin_unreachable(); } while (false);
return operator[](this->size() - 1);
}
[[__nodiscard__]]
const_reference
back() const noexcept
{
do { if (std::__is_constant_evaluated() && !bool(!empty())) __builtin_unreachable(); } while (false);
return operator[](this->size() - 1);
}
basic_string&
operator+=(const basic_string& __str)
{ return this->append(__str); }
basic_string&
operator+=(const _CharT* __s)
{ return this->append(__s); }
basic_string&
operator+=(_CharT __c)
{
this->push_back(__c);
return *this;
}
basic_string&
operator+=(initializer_list<_CharT> __l)
{ return this->append(__l.begin(), __l.size()); }
template<typename _Tp>
_If_sv<_Tp, basic_string&>
operator+=(const _Tp& __svt)
{ return this->append(__svt); }
basic_string&
append(const basic_string& __str)
{ return this->append(__str._M_data(), __str.size()); }
basic_string&
append(const basic_string& __str, size_type __pos, size_type __n = npos)
{ return this->append(__str._M_data()
+ __str._M_check(__pos, "basic_string::append"),
__str._M_limit(__pos, __n)); }
basic_string&
append(const _CharT* __s, size_type __n)
{
;
_M_check_length(size_type(0), __n, "basic_string::append");
return _M_append(__s, __n);
}
basic_string&
append(const _CharT* __s)
{
;
const size_type __n = traits_type::length(__s);
_M_check_length(size_type(0), __n, "basic_string::append");
return _M_append(__s, __n);
}
basic_string&
append(size_type __n, _CharT __c)
{ return _M_replace_aux(this->size(), size_type(0), __n, __c); }
basic_string&
append(initializer_list<_CharT> __l)
{ return this->append(__l.begin(), __l.size()); }
template<class _InputIterator,
typename = std::_RequireInputIter<_InputIterator>>
basic_string&
append(_InputIterator __first, _InputIterator __last)
{ return this->replace(end(), end(), __first, __last); }
template<typename _Tp>
_If_sv<_Tp, basic_string&>
append(const _Tp& __svt)
{
__sv_type __sv = __svt;
return this->append(__sv.data(), __sv.size());
}
template<typename _Tp>
_If_sv<_Tp, basic_string&>
append(const _Tp& __svt, size_type __pos, size_type __n = npos)
{
__sv_type __sv = __svt;
return _M_append(__sv.data()
+ std::__sv_check(__sv.size(), __pos, "basic_string::append"),
std::__sv_limit(__sv.size(), __pos, __n));
}
void
push_back(_CharT __c)
{
const size_type __size = this->size();
if (__size + 1 > this->capacity())
this->_M_mutate(__size, size_type(0), 0, size_type(1));
traits_type::assign(this->_M_data()[__size], __c);
this->_M_set_length(__size + 1);
}
basic_string&
assign(const basic_string& __str)
{
if (_Alloc_traits::_S_propagate_on_copy_assign())
{
if (!_Alloc_traits::_S_always_equal() && !_M_is_local()
&& _M_get_allocator() != __str._M_get_allocator())
{
if (__str.size() <= _S_local_capacity)
{
_M_destroy(_M_allocated_capacity);
_M_data(_M_use_local_data());
_M_set_length(0);
}
else
{
const auto __len = __str.size();
auto __alloc = __str._M_get_allocator();
auto __ptr = _S_allocate(__alloc, __len + 1);
_M_destroy(_M_allocated_capacity);
_M_data(__ptr);
_M_capacity(__len);
_M_set_length(__len);
}
}
std::__alloc_on_copy(_M_get_allocator(), __str._M_get_allocator());
}
this->_M_assign(__str);
return *this;
}
basic_string&
assign(basic_string&& __str)
noexcept(_Alloc_traits::_S_nothrow_move())
{
return *this = std::move(__str);
}
basic_string&
assign(const basic_string& __str, size_type __pos, size_type __n = npos)
{ return _M_replace(size_type(0), this->size(), __str._M_data()
+ __str._M_check(__pos, "basic_string::assign"),
__str._M_limit(__pos, __n)); }
basic_string&
assign(const _CharT* __s, size_type __n)
{
;
return _M_replace(size_type(0), this->size(), __s, __n);
}
basic_string&
assign(const _CharT* __s)
{
;
return _M_replace(size_type(0), this->size(), __s,
traits_type::length(__s));
}
basic_string&
assign(size_type __n, _CharT __c)
{ return _M_replace_aux(size_type(0), this->size(), __n, __c); }
template<class _InputIterator,
typename = std::_RequireInputIter<_InputIterator>>
basic_string&
assign(_InputIterator __first, _InputIterator __last)
{ return this->replace(begin(), end(), __first, __last); }
basic_string&
assign(initializer_list<_CharT> __l)
{ return this->assign(__l.begin(), __l.size()); }
template<typename _Tp>
_If_sv<_Tp, basic_string&>
assign(const _Tp& __svt)
{
__sv_type __sv = __svt;
return this->assign(__sv.data(), __sv.size());
}
template<typename _Tp>
_If_sv<_Tp, basic_string&>
assign(const _Tp& __svt, size_type __pos, size_type __n = npos)
{
__sv_type __sv = __svt;
return _M_replace(size_type(0), this->size(),
__sv.data()
+ std::__sv_check(__sv.size(), __pos, "basic_string::assign"),
std::__sv_limit(__sv.size(), __pos, __n));
}
iterator
insert(const_iterator __p, size_type __n, _CharT __c)
{
;
const size_type __pos = __p - begin();
this->replace(__p, __p, __n, __c);
return iterator(this->_M_data() + __pos);
}
template<class _InputIterator,
typename = std::_RequireInputIter<_InputIterator>>
iterator
insert(const_iterator __p, _InputIterator __beg, _InputIterator __end)
{
;
const size_type __pos = __p - begin();
this->replace(__p, __p, __beg, __end);
return iterator(this->_M_data() + __pos);
}
iterator
insert(const_iterator __p, initializer_list<_CharT> __l)
{ return this->insert(__p, __l.begin(), __l.end()); }
basic_string&
insert(size_type __pos1, const basic_string& __str)
{ return this->replace(__pos1, size_type(0),
__str._M_data(), __str.size()); }
basic_string&
insert(size_type __pos1, const basic_string& __str,
size_type __pos2, size_type __n = npos)
{ return this->replace(__pos1, size_type(0), __str._M_data()
+ __str._M_check(__pos2, "basic_string::insert"),
__str._M_limit(__pos2, __n)); }
basic_string&
insert(size_type __pos, const _CharT* __s, size_type __n)
{ return this->replace(__pos, size_type(0), __s, __n); }
basic_string&
insert(size_type __pos, const _CharT* __s)
{
;
return this->replace(__pos, size_type(0), __s,
traits_type::length(__s));
}
basic_string&
insert(size_type __pos, size_type __n, _CharT __c)
{ return _M_replace_aux(_M_check(__pos, "basic_string::insert"),
size_type(0), __n, __c); }
iterator
insert(__const_iterator __p, _CharT __c)
{
;
const size_type __pos = __p - begin();
_M_replace_aux(__pos, size_type(0), size_type(1), __c);
return iterator(_M_data() + __pos);
}
template<typename _Tp>
_If_sv<_Tp, basic_string&>
insert(size_type __pos, const _Tp& __svt)
{
__sv_type __sv = __svt;
return this->insert(__pos, __sv.data(), __sv.size());
}
template<typename _Tp>
_If_sv<_Tp, basic_string&>
insert(size_type __pos1, const _Tp& __svt,
size_type __pos2, size_type __n = npos)
{
__sv_type __sv = __svt;
return this->replace(__pos1, size_type(0),
__sv.data()
+ std::__sv_check(__sv.size(), __pos2, "basic_string::insert"),
std::__sv_limit(__sv.size(), __pos2, __n));
}
basic_string&
erase(size_type __pos = 0, size_type __n = npos)
{
_M_check(__pos, "basic_string::erase");
if (__n == npos)
this->_M_set_length(__pos);
else if (__n != 0)
this->_M_erase(__pos, _M_limit(__pos, __n));
return *this;
}
iterator
erase(__const_iterator __position)
{
;
const size_type __pos = __position - begin();
this->_M_erase(__pos, size_type(1));
return iterator(_M_data() + __pos);
}
iterator
erase(__const_iterator __first, __const_iterator __last)
{
;
const size_type __pos = __first - begin();
if (__last == end())
this->_M_set_length(__pos);
else
this->_M_erase(__pos, __last - __first);
return iterator(this->_M_data() + __pos);
}
void
pop_back() noexcept
{
do { if (std::__is_constant_evaluated() && !bool(!empty())) __builtin_unreachable(); } while (false);
_M_erase(size() - 1, 1);
}
basic_string&
replace(size_type __pos, size_type __n, const basic_string& __str)
{ return this->replace(__pos, __n, __str._M_data(), __str.size()); }
basic_string&
replace(size_type __pos1, size_type __n1, const basic_string& __str,
size_type __pos2, size_type __n2 = npos)
{ return this->replace(__pos1, __n1, __str._M_data()
+ __str._M_check(__pos2, "basic_string::replace"),
__str._M_limit(__pos2, __n2)); }
basic_string&
replace(size_type __pos, size_type __n1, const _CharT* __s,
size_type __n2)
{
;
return _M_replace(_M_check(__pos, "basic_string::replace"),
_M_limit(__pos, __n1), __s, __n2);
}
basic_string&
replace(size_type __pos, size_type __n1, const _CharT* __s)
{
;
return this->replace(__pos, __n1, __s, traits_type::length(__s));
}
basic_string&
replace(size_type __pos, size_type __n1, size_type __n2, _CharT __c)
{ return _M_replace_aux(_M_check(__pos, "basic_string::replace"),
_M_limit(__pos, __n1), __n2, __c); }
basic_string&
replace(__const_iterator __i1, __const_iterator __i2,
const basic_string& __str)
{ return this->replace(__i1, __i2, __str._M_data(), __str.size()); }
basic_string&
replace(__const_iterator __i1, __const_iterator __i2,
const _CharT* __s, size_type __n)
{
;
return this->replace(__i1 - begin(), __i2 - __i1, __s, __n);
}
basic_string&
replace(__const_iterator __i1, __const_iterator __i2, const _CharT* __s)
{
;
return this->replace(__i1, __i2, __s, traits_type::length(__s));
}
basic_string&
replace(__const_iterator __i1, __const_iterator __i2, size_type __n,
_CharT __c)
{
;
return _M_replace_aux(__i1 - begin(), __i2 - __i1, __n, __c);
}
template<class _InputIterator,
typename = std::_RequireInputIter<_InputIterator>>
basic_string&
replace(const_iterator __i1, const_iterator __i2,
_InputIterator __k1, _InputIterator __k2)
{
;
;
return this->_M_replace_dispatch(__i1, __i2, __k1, __k2,
std::__false_type());
}
basic_string&
replace(__const_iterator __i1, __const_iterator __i2,
_CharT* __k1, _CharT* __k2)
{
;
;
return this->replace(__i1 - begin(), __i2 - __i1,
__k1, __k2 - __k1);
}
basic_string&
replace(__const_iterator __i1, __const_iterator __i2,
const _CharT* __k1, const _CharT* __k2)
{
;
;
return this->replace(__i1 - begin(), __i2 - __i1,
__k1, __k2 - __k1);
}
basic_string&
replace(__const_iterator __i1, __const_iterator __i2,
iterator __k1, iterator __k2)
{
;
;
return this->replace(__i1 - begin(), __i2 - __i1,
__k1.base(), __k2 - __k1);
}
basic_string&
replace(__const_iterator __i1, __const_iterator __i2,
const_iterator __k1, const_iterator __k2)
{
;
;
return this->replace(__i1 - begin(), __i2 - __i1,
__k1.base(), __k2 - __k1);
}
basic_string& replace(const_iterator __i1, const_iterator __i2,
initializer_list<_CharT> __l)
{ return this->replace(__i1, __i2, __l.begin(), __l.size()); }
template<typename _Tp>
_If_sv<_Tp, basic_string&>
replace(size_type __pos, size_type __n, const _Tp& __svt)
{
__sv_type __sv = __svt;
return this->replace(__pos, __n, __sv.data(), __sv.size());
}
template<typename _Tp>
_If_sv<_Tp, basic_string&>
replace(size_type __pos1, size_type __n1, const _Tp& __svt,
size_type __pos2, size_type __n2 = npos)
{
__sv_type __sv = __svt;
return this->replace(__pos1, __n1,
__sv.data()
+ std::__sv_check(__sv.size(), __pos2, "basic_string::replace"),
std::__sv_limit(__sv.size(), __pos2, __n2));
}
template<typename _Tp>
_If_sv<_Tp, basic_string&>
replace(const_iterator __i1, const_iterator __i2, const _Tp& __svt)
{
__sv_type __sv = __svt;
return this->replace(__i1 - begin(), __i2 - __i1, __sv);
}
private:
template<class _Integer>
basic_string&
_M_replace_dispatch(const_iterator __i1, const_iterator __i2,
_Integer __n, _Integer __val, __true_type)
{ return _M_replace_aux(__i1 - begin(), __i2 - __i1, __n, __val); }
template<class _InputIterator>
basic_string&
_M_replace_dispatch(const_iterator __i1, const_iterator __i2,
_InputIterator __k1, _InputIterator __k2,
__false_type);
basic_string&
_M_replace_aux(size_type __pos1, size_type __n1, size_type __n2,
_CharT __c);
__attribute__((__noinline__, __noclone__, __cold__)) void
_M_replace_cold(pointer __p, size_type __len1, const _CharT* __s,
const size_type __len2, const size_type __how_much);
basic_string&
_M_replace(size_type __pos, size_type __len1, const _CharT* __s,
const size_type __len2);
basic_string&
_M_append(const _CharT* __s, size_type __n);
public:
size_type
copy(_CharT* __s, size_type __n, size_type __pos = 0) const;
void
swap(basic_string& __s) noexcept;
[[__nodiscard__]]
const _CharT*
c_str() const noexcept
{ return _M_data(); }
[[__nodiscard__]]
const _CharT*
data() const noexcept
{ return _M_data(); }
[[__nodiscard__]]
_CharT*
data() noexcept
{ return _M_data(); }
[[__nodiscard__]]
allocator_type
get_allocator() const noexcept
{ return _M_get_allocator(); }
[[__nodiscard__]]
size_type
find(const _CharT* __s, size_type __pos, size_type __n) const
noexcept;
[[__nodiscard__]]
size_type
find(const basic_string& __str, size_type __pos = 0) const
noexcept
{ return this->find(__str.data(), __pos, __str.size()); }
template<typename _Tp>
[[__nodiscard__]]
_If_sv<_Tp, size_type>
find(const _Tp& __svt, size_type __pos = 0) const
noexcept(is_same<_Tp, __sv_type>::value)
{
__sv_type __sv = __svt;
return this->find(__sv.data(), __pos, __sv.size());
}
[[__nodiscard__]]
size_type
find(const _CharT* __s, size_type __pos = 0) const noexcept
{
;
return this->find(__s, __pos, traits_type::length(__s));
}
[[__nodiscard__]]
size_type
find(_CharT __c, size_type __pos = 0) const noexcept;
[[__nodiscard__]]
size_type
rfind(const basic_string& __str, size_type __pos = npos) const
noexcept
{ return this->rfind(__str.data(), __pos, __str.size()); }
template<typename _Tp>
[[__nodiscard__]]
_If_sv<_Tp, size_type>
rfind(const _Tp& __svt, size_type __pos = npos) const
noexcept(is_same<_Tp, __sv_type>::value)
{
__sv_type __sv = __svt;
return this->rfind(__sv.data(), __pos, __sv.size());
}
[[__nodiscard__]]
size_type
rfind(const _CharT* __s, size_type __pos, size_type __n) const
noexcept;
[[__nodiscard__]]
size_type
rfind(const _CharT* __s, size_type __pos = npos) const
{
;
return this->rfind(__s, __pos, traits_type::length(__s));
}
[[__nodiscard__]]
size_type
rfind(_CharT __c, size_type __pos = npos) const noexcept;
[[__nodiscard__]]
size_type
find_first_of(const basic_string& __str, size_type __pos = 0) const
noexcept
{ return this->find_first_of(__str.data(), __pos, __str.size()); }
template<typename _Tp>
[[__nodiscard__]]
_If_sv<_Tp, size_type>
find_first_of(const _Tp& __svt, size_type __pos = 0) const
noexcept(is_same<_Tp, __sv_type>::value)
{
__sv_type __sv = __svt;
return this->find_first_of(__sv.data(), __pos, __sv.size());
}
[[__nodiscard__]]
size_type
find_first_of(const _CharT* __s, size_type __pos, size_type __n) const
noexcept;
[[__nodiscard__]]
size_type
find_first_of(const _CharT* __s, size_type __pos = 0) const
noexcept
{
;
return this->find_first_of(__s, __pos, traits_type::length(__s));
}
[[__nodiscard__]]
size_type
find_first_of(_CharT __c, size_type __pos = 0) const noexcept
{ return this->find(__c, __pos); }
[[__nodiscard__]]
size_type
find_last_of(const basic_string& __str, size_type __pos = npos) const
noexcept
{ return this->find_last_of(__str.data(), __pos, __str.size()); }
template<typename _Tp>
[[__nodiscard__]]
_If_sv<_Tp, size_type>
find_last_of(const _Tp& __svt, size_type __pos = npos) const
noexcept(is_same<_Tp, __sv_type>::value)
{
__sv_type __sv = __svt;
return this->find_last_of(__sv.data(), __pos, __sv.size());
}
[[__nodiscard__]]
size_type
find_last_of(const _CharT* __s, size_type __pos, size_type __n) const
noexcept;
[[__nodiscard__]]
size_type
find_last_of(const _CharT* __s, size_type __pos = npos) const
noexcept
{
;
return this->find_last_of(__s, __pos, traits_type::length(__s));
}
[[__nodiscard__]]
size_type
find_last_of(_CharT __c, size_type __pos = npos) const noexcept
{ return this->rfind(__c, __pos); }
[[__nodiscard__]]
size_type
find_first_not_of(const basic_string& __str, size_type __pos = 0) const
noexcept
{ return this->find_first_not_of(__str.data(), __pos, __str.size()); }
template<typename _Tp>
[[__nodiscard__]]
_If_sv<_Tp, size_type>
find_first_not_of(const _Tp& __svt, size_type __pos = 0) const
noexcept(is_same<_Tp, __sv_type>::value)
{
__sv_type __sv = __svt;
return this->find_first_not_of(__sv.data(), __pos, __sv.size());
}
[[__nodiscard__]]
size_type
find_first_not_of(const _CharT* __s, size_type __pos,
size_type __n) const noexcept;
[[__nodiscard__]]
size_type
find_first_not_of(const _CharT* __s, size_type __pos = 0) const
noexcept
{
;
return this->find_first_not_of(__s, __pos, traits_type::length(__s));
}
[[__nodiscard__]]
size_type
find_first_not_of(_CharT __c, size_type __pos = 0) const
noexcept;
[[__nodiscard__]]
size_type
find_last_not_of(const basic_string& __str, size_type __pos = npos) const
noexcept
{ return this->find_last_not_of(__str.data(), __pos, __str.size()); }
template<typename _Tp>
[[__nodiscard__]]
_If_sv<_Tp, size_type>
find_last_not_of(const _Tp& __svt, size_type __pos = npos) const
noexcept(is_same<_Tp, __sv_type>::value)
{
__sv_type __sv = __svt;
return this->find_last_not_of(__sv.data(), __pos, __sv.size());
}
[[__nodiscard__]]
size_type
find_last_not_of(const _CharT* __s, size_type __pos,
size_type __n) const noexcept;
[[__nodiscard__]]
size_type
find_last_not_of(const _CharT* __s, size_type __pos = npos) const
noexcept
{
;
return this->find_last_not_of(__s, __pos, traits_type::length(__s));
}
[[__nodiscard__]]
size_type
find_last_not_of(_CharT __c, size_type __pos = npos) const
noexcept;
[[__nodiscard__]]
basic_string
substr(size_type __pos = 0, size_type __n = npos) const
{ return basic_string(*this,
_M_check(__pos, "basic_string::substr"), __n); }
[[__nodiscard__]]
int
compare(const basic_string& __str) const
{
const size_type __size = this->size();
const size_type __osize = __str.size();
const size_type __len = std::min(__size, __osize);
int __r = traits_type::compare(_M_data(), __str.data(), __len);
if (!__r)
__r = _S_compare(__size, __osize);
return __r;
}
template<typename _Tp>
[[__nodiscard__]]
_If_sv<_Tp, int>
compare(const _Tp& __svt) const
noexcept(is_same<_Tp, __sv_type>::value)
{
__sv_type __sv = __svt;
const size_type __size = this->size();
const size_type __osize = __sv.size();
const size_type __len = std::min(__size, __osize);
int __r = traits_type::compare(_M_data(), __sv.data(), __len);
if (!__r)
__r = _S_compare(__size, __osize);
return __r;
}
template<typename _Tp>
[[__nodiscard__]]
_If_sv<_Tp, int>
compare(size_type __pos, size_type __n, const _Tp& __svt) const
noexcept(is_same<_Tp, __sv_type>::value)
{
__sv_type __sv = __svt;
return __sv_type(*this).substr(__pos, __n).compare(__sv);
}
template<typename _Tp>
[[__nodiscard__]]
_If_sv<_Tp, int>
compare(size_type __pos1, size_type __n1, const _Tp& __svt,
size_type __pos2, size_type __n2 = npos) const
noexcept(is_same<_Tp, __sv_type>::value)
{
__sv_type __sv = __svt;
return __sv_type(*this)
.substr(__pos1, __n1).compare(__sv.substr(__pos2, __n2));
}
[[__nodiscard__]]
int
compare(size_type __pos, size_type __n, const basic_string& __str) const
{
_M_check(__pos, "basic_string::compare");
__n = _M_limit(__pos, __n);
const size_type __osize = __str.size();
const size_type __len = std::min(__n, __osize);
int __r = traits_type::compare(_M_data() + __pos, __str.data(), __len);
if (!__r)
__r = _S_compare(__n, __osize);
return __r;
}
[[__nodiscard__]]
int
compare(size_type __pos1, size_type __n1, const basic_string& __str,
size_type __pos2, size_type __n2 = npos) const
{
_M_check(__pos1, "basic_string::compare");
__str._M_check(__pos2, "basic_string::compare");
__n1 = _M_limit(__pos1, __n1);
__n2 = __str._M_limit(__pos2, __n2);
const size_type __len = std::min(__n1, __n2);
int __r = traits_type::compare(_M_data() + __pos1,
__str.data() + __pos2, __len);
if (!__r)
__r = _S_compare(__n1, __n2);
return __r;
}
[[__nodiscard__]]
int
compare(const _CharT* __s) const noexcept
{
;
const size_type __size = this->size();
const size_type __osize = traits_type::length(__s);
const size_type __len = std::min(__size, __osize);
int __r = traits_type::compare(_M_data(), __s, __len);
if (!__r)
__r = _S_compare(__size, __osize);
return __r;
}
[[__nodiscard__]]
int
compare(size_type __pos, size_type __n1, const _CharT* __s) const
{
;
_M_check(__pos, "basic_string::compare");
__n1 = _M_limit(__pos, __n1);
const size_type __osize = traits_type::length(__s);
const size_type __len = std::min(__n1, __osize);
int __r = traits_type::compare(_M_data() + __pos, __s, __len);
if (!__r)
__r = _S_compare(__n1, __osize);
return __r;
}
[[__nodiscard__]]
int
compare(size_type __pos, size_type __n1, const _CharT* __s,
size_type __n2) const
{
;
_M_check(__pos, "basic_string::compare");
__n1 = _M_limit(__pos, __n1);
const size_type __len = std::min(__n1, __n2);
int __r = traits_type::compare(_M_data() + __pos, __s, __len);
if (!__r)
__r = _S_compare(__n1, __n2);
return __r;
}
template<typename, typename, typename> friend class basic_stringbuf;
};
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __cxx11 {
template<typename _InputIterator, typename _CharT
= typename iterator_traits<_InputIterator>::value_type,
typename _Allocator = allocator<_CharT>,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireAllocator<_Allocator>>
basic_string(_InputIterator, _InputIterator, _Allocator = _Allocator())
-> basic_string<_CharT, char_traits<_CharT>, _Allocator>;
template<typename _CharT, typename _Traits,
typename _Allocator = allocator<_CharT>,
typename = _RequireAllocator<_Allocator>>
basic_string(basic_string_view<_CharT, _Traits>, const _Allocator& = _Allocator())
-> basic_string<_CharT, _Traits, _Allocator>;
template<typename _CharT, typename _Traits,
typename _Allocator = allocator<_CharT>,
typename = _RequireAllocator<_Allocator>>
basic_string(basic_string_view<_CharT, _Traits>,
typename basic_string<_CharT, _Traits, _Allocator>::size_type,
typename basic_string<_CharT, _Traits, _Allocator>::size_type,
const _Allocator& = _Allocator())
-> basic_string<_CharT, _Traits, _Allocator>;
}
template<typename _Str>
inline _Str
__str_concat(typename _Str::value_type const* __lhs,
typename _Str::size_type __lhs_len,
typename _Str::value_type const* __rhs,
typename _Str::size_type __rhs_len,
typename _Str::allocator_type const& __a)
{
typedef typename _Str::allocator_type allocator_type;
typedef __gnu_cxx::__alloc_traits<allocator_type> _Alloc_traits;
_Str __str(_Alloc_traits::_S_select_on_copy(__a));
__str.reserve(__lhs_len + __rhs_len);
__str.append(__lhs, __lhs_len);
__str.append(__rhs, __rhs_len);
return __str;
}
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline basic_string<_CharT, _Traits, _Alloc>
operator+(const basic_string<_CharT, _Traits, _Alloc>& __lhs,
const basic_string<_CharT, _Traits, _Alloc>& __rhs)
{
typedef basic_string<_CharT, _Traits, _Alloc> _Str;
return std::__str_concat<_Str>(__lhs.c_str(), __lhs.size(),
__rhs.c_str(), __rhs.size(),
__lhs.get_allocator());
}
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline basic_string<_CharT,_Traits,_Alloc>
operator+(const _CharT* __lhs,
const basic_string<_CharT,_Traits,_Alloc>& __rhs)
{
;
typedef basic_string<_CharT, _Traits, _Alloc> _Str;
return std::__str_concat<_Str>(__lhs, _Traits::length(__lhs),
__rhs.c_str(), __rhs.size(),
__rhs.get_allocator());
}
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline basic_string<_CharT,_Traits,_Alloc>
operator+(_CharT __lhs, const basic_string<_CharT,_Traits,_Alloc>& __rhs)
{
typedef basic_string<_CharT, _Traits, _Alloc> _Str;
return std::__str_concat<_Str>(__builtin_addressof(__lhs), 1,
__rhs.c_str(), __rhs.size(),
__rhs.get_allocator());
}
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline basic_string<_CharT, _Traits, _Alloc>
operator+(const basic_string<_CharT, _Traits, _Alloc>& __lhs,
const _CharT* __rhs)
{
;
typedef basic_string<_CharT, _Traits, _Alloc> _Str;
return std::__str_concat<_Str>(__lhs.c_str(), __lhs.size(),
__rhs, _Traits::length(__rhs),
__lhs.get_allocator());
}
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline basic_string<_CharT, _Traits, _Alloc>
operator+(const basic_string<_CharT, _Traits, _Alloc>& __lhs, _CharT __rhs)
{
typedef basic_string<_CharT, _Traits, _Alloc> _Str;
return std::__str_concat<_Str>(__lhs.c_str(), __lhs.size(),
__builtin_addressof(__rhs), 1,
__lhs.get_allocator());
}
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline basic_string<_CharT, _Traits, _Alloc>
operator+(basic_string<_CharT, _Traits, _Alloc>&& __lhs,
const basic_string<_CharT, _Traits, _Alloc>& __rhs)
{ return std::move(__lhs.append(__rhs)); }
template<typename _CharT, typename _Traits, typename _Alloc>
inline basic_string<_CharT, _Traits, _Alloc>
operator+(const basic_string<_CharT, _Traits, _Alloc>& __lhs,
basic_string<_CharT, _Traits, _Alloc>&& __rhs)
{ return std::move(__rhs.insert(0, __lhs)); }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline basic_string<_CharT, _Traits, _Alloc>
operator+(basic_string<_CharT, _Traits, _Alloc>&& __lhs,
basic_string<_CharT, _Traits, _Alloc>&& __rhs)
{
using _Alloc_traits = allocator_traits<_Alloc>;
bool __use_rhs = false;
if constexpr (typename _Alloc_traits::is_always_equal{})
__use_rhs = true;
else if (__lhs.get_allocator() == __rhs.get_allocator())
__use_rhs = true;
if (__use_rhs)
{
const auto __size = __lhs.size() + __rhs.size();
if (__size > __lhs.capacity() && __size <= __rhs.capacity())
return std::move(__rhs.insert(0, __lhs));
}
return std::move(__lhs.append(__rhs));
}
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]] [[__nodiscard__]]
inline basic_string<_CharT, _Traits, _Alloc>
operator+(const _CharT* __lhs,
basic_string<_CharT, _Traits, _Alloc>&& __rhs)
{ return std::move(__rhs.insert(0, __lhs)); }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline basic_string<_CharT, _Traits, _Alloc>
operator+(_CharT __lhs,
basic_string<_CharT, _Traits, _Alloc>&& __rhs)
{ return std::move(__rhs.insert(0, 1, __lhs)); }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline basic_string<_CharT, _Traits, _Alloc>
operator+(basic_string<_CharT, _Traits, _Alloc>&& __lhs,
const _CharT* __rhs)
{ return std::move(__lhs.append(__rhs)); }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline basic_string<_CharT, _Traits, _Alloc>
operator+(basic_string<_CharT, _Traits, _Alloc>&& __lhs,
_CharT __rhs)
{ return std::move(__lhs.append(1, __rhs)); }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator==(const basic_string<_CharT, _Traits, _Alloc>& __lhs,
const basic_string<_CharT, _Traits, _Alloc>& __rhs)
noexcept
{
return __lhs.size() == __rhs.size()
&& !_Traits::compare(__lhs.data(), __rhs.data(), __lhs.size());
}
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator==(const basic_string<_CharT, _Traits, _Alloc>& __lhs,
const _CharT* __rhs)
{
return __lhs.size() == _Traits::length(__rhs)
&& !_Traits::compare(__lhs.data(), __rhs, __lhs.size());
}
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator==(const _CharT* __lhs,
const basic_string<_CharT, _Traits, _Alloc>& __rhs)
{ return __rhs == __lhs; }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator!=(const basic_string<_CharT, _Traits, _Alloc>& __lhs,
const basic_string<_CharT, _Traits, _Alloc>& __rhs)
noexcept
{ return !(__lhs == __rhs); }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator!=(const _CharT* __lhs,
const basic_string<_CharT, _Traits, _Alloc>& __rhs)
{ return !(__rhs == __lhs); }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator!=(const basic_string<_CharT, _Traits, _Alloc>& __lhs,
const _CharT* __rhs)
{ return !(__lhs == __rhs); }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator<(const basic_string<_CharT, _Traits, _Alloc>& __lhs,
const basic_string<_CharT, _Traits, _Alloc>& __rhs)
noexcept
{ return __lhs.compare(__rhs) < 0; }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator<(const basic_string<_CharT, _Traits, _Alloc>& __lhs,
const _CharT* __rhs)
{ return __lhs.compare(__rhs) < 0; }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator<(const _CharT* __lhs,
const basic_string<_CharT, _Traits, _Alloc>& __rhs)
{ return __rhs.compare(__lhs) > 0; }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator>(const basic_string<_CharT, _Traits, _Alloc>& __lhs,
const basic_string<_CharT, _Traits, _Alloc>& __rhs)
noexcept
{ return __lhs.compare(__rhs) > 0; }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator>(const basic_string<_CharT, _Traits, _Alloc>& __lhs,
const _CharT* __rhs)
{ return __lhs.compare(__rhs) > 0; }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator>(const _CharT* __lhs,
const basic_string<_CharT, _Traits, _Alloc>& __rhs)
{ return __rhs.compare(__lhs) < 0; }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator<=(const basic_string<_CharT, _Traits, _Alloc>& __lhs,
const basic_string<_CharT, _Traits, _Alloc>& __rhs)
noexcept
{ return __lhs.compare(__rhs) <= 0; }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator<=(const basic_string<_CharT, _Traits, _Alloc>& __lhs,
const _CharT* __rhs)
{ return __lhs.compare(__rhs) <= 0; }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator<=(const _CharT* __lhs,
const basic_string<_CharT, _Traits, _Alloc>& __rhs)
{ return __rhs.compare(__lhs) >= 0; }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator>=(const basic_string<_CharT, _Traits, _Alloc>& __lhs,
const basic_string<_CharT, _Traits, _Alloc>& __rhs)
noexcept
{ return __lhs.compare(__rhs) >= 0; }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator>=(const basic_string<_CharT, _Traits, _Alloc>& __lhs,
const _CharT* __rhs)
{ return __lhs.compare(__rhs) >= 0; }
template<typename _CharT, typename _Traits, typename _Alloc>
[[__nodiscard__]]
inline bool
operator>=(const _CharT* __lhs,
const basic_string<_CharT, _Traits, _Alloc>& __rhs)
{ return __rhs.compare(__lhs) <= 0; }
template<typename _CharT, typename _Traits, typename _Alloc>
inline void
swap(basic_string<_CharT, _Traits, _Alloc>& __lhs,
basic_string<_CharT, _Traits, _Alloc>& __rhs)
noexcept(noexcept(__lhs.swap(__rhs)))
{ __lhs.swap(__rhs); }
template<typename _CharT, typename _Traits, typename _Alloc>
basic_istream<_CharT, _Traits>&
operator>>(basic_istream<_CharT, _Traits>& __is,
basic_string<_CharT, _Traits, _Alloc>& __str);
template<>
basic_istream<char>&
operator>>(basic_istream<char>& __is, basic_string<char>& __str);
template<typename _CharT, typename _Traits, typename _Alloc>
inline basic_ostream<_CharT, _Traits>&
operator<<(basic_ostream<_CharT, _Traits>& __os,
const basic_string<_CharT, _Traits, _Alloc>& __str)
{
return __ostream_insert(__os, __str.data(), __str.size());
}
template<typename _CharT, typename _Traits, typename _Alloc>
basic_istream<_CharT, _Traits>&
getline(basic_istream<_CharT, _Traits>& __is,
basic_string<_CharT, _Traits, _Alloc>& __str, _CharT __delim);
template<typename _CharT, typename _Traits, typename _Alloc>
inline basic_istream<_CharT, _Traits>&
getline(basic_istream<_CharT, _Traits>& __is,
basic_string<_CharT, _Traits, _Alloc>& __str)
{ return std::getline(__is, __str, __is.widen('\n')); }
template<typename _CharT, typename _Traits, typename _Alloc>
inline basic_istream<_CharT, _Traits>&
getline(basic_istream<_CharT, _Traits>&& __is,
basic_string<_CharT, _Traits, _Alloc>& __str, _CharT __delim)
{ return std::getline(__is, __str, __delim); }
template<typename _CharT, typename _Traits, typename _Alloc>
inline basic_istream<_CharT, _Traits>&
getline(basic_istream<_CharT, _Traits>&& __is,
basic_string<_CharT, _Traits, _Alloc>& __str)
{ return std::getline(__is, __str); }
template<>
basic_istream<char>&
getline(basic_istream<char>& __in, basic_string<char>& __str,
char __delim);
template<>
basic_istream<wchar_t>&
getline(basic_istream<wchar_t>& __in, basic_string<wchar_t>& __str,
wchar_t __delim);
}
namespace std
{
using ::FILE;
using ::fpos_t;
using ::clearerr;
using ::fclose;
using ::feof;
using ::ferror;
using ::fflush;
using ::fgetc;
using ::fgetpos;
using ::fgets;
using ::fopen;
using ::fprintf;
using ::fputc;
using ::fputs;
using ::fread;
using ::freopen;
using ::fscanf;
using ::fseek;
using ::fsetpos;
using ::ftell;
using ::fwrite;
using ::getc;
using ::getchar;
using ::perror;
using ::printf;
using ::putc;
using ::putchar;
using ::puts;
using ::remove;
using ::rename;
using ::rewind;
using ::scanf;
using ::setbuf;
using ::setvbuf;
using ::sprintf;
using ::sscanf;
using ::tmpfile;
using ::tmpnam;
using ::ungetc;
using ::vfprintf;
using ::vprintf;
using ::vsprintf;
}
namespace __gnu_cxx
{
using ::snprintf;
using ::vfscanf;
using ::vscanf;
using ::vsnprintf;
using ::vsscanf;
}
namespace std
{
using ::__gnu_cxx::snprintf;
using ::__gnu_cxx::vfscanf;
using ::__gnu_cxx::vscanf;
using ::__gnu_cxx::vsnprintf;
using ::__gnu_cxx::vsscanf;
}
extern "C" {
extern int *__errno_location (void) noexcept (true) __attribute__ ((__const__));
extern char *program_invocation_name;
extern char *program_invocation_short_name;
typedef int error_t;
}
namespace __gnu_cxx __attribute__ ((__visibility__ ("default")))
{
template<typename _TRet, typename _Ret = _TRet, typename _CharT,
typename... _Base>
_Ret
__stoa(_TRet (*__convf) (const _CharT*, _CharT**, _Base...),
const char* __name, const _CharT* __str, std::size_t* __idx,
_Base... __base)
{
_Ret __ret;
_CharT* __endptr;
struct _Save_errno {
_Save_errno() : _M_errno((*__errno_location ())) { (*__errno_location ()) = 0; }
~_Save_errno() { if ((*__errno_location ()) == 0) (*__errno_location ()) = _M_errno; }
int _M_errno;
} const __save_errno;
struct _Range_chk {
static bool
_S_chk(_TRet, std::false_type) { return false; }
static bool
_S_chk(_TRet __val, std::true_type)
{
return __val < _TRet(__numeric_traits<int>::__min)
|| __val > _TRet(__numeric_traits<int>::__max);
}
};
const _TRet __tmp = __convf(__str, &__endptr, __base...);
if (__endptr == __str)
std::__throw_invalid_argument(__name);
else if ((*__errno_location ()) == 34
|| _Range_chk::_S_chk(__tmp, std::is_same<_Ret, int>{}))
std::__throw_out_of_range(__name);
else
__ret = __tmp;
if (__idx)
*__idx = __endptr - __str;
return __ret;
}
template<typename _String, typename _CharT = typename _String::value_type>
_String
__to_xstring(int (*__convf) (_CharT*, std::size_t, const _CharT*,
__builtin_va_list), std::size_t __n,
const _CharT* __fmt, ...)
{
_CharT* __s = static_cast<_CharT*>(__builtin_alloca(sizeof(_CharT)
* __n));
__builtin_va_list __args;
__builtin_va_start(__args, __fmt);
const int __len = __convf(__s, __n, __fmt, __args);
__builtin_va_end(__args);
return _String(__s, __s + __len);
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __detail
{
template<typename _Tp>
constexpr bool __integer_to_chars_is_unsigned
= ! __gnu_cxx::__int_traits<_Tp>::__is_signed;
template<typename _Tp>
constexpr unsigned
__to_chars_len(_Tp __value, int __base = 10) noexcept
{
static_assert(__integer_to_chars_is_unsigned<_Tp>, "implementation bug");
unsigned __n = 1;
const unsigned __b2 = __base * __base;
const unsigned __b3 = __b2 * __base;
const unsigned long __b4 = __b3 * __base;
for (;;)
{
if (__value < (unsigned)__base) return __n;
if (__value < __b2) return __n + 1;
if (__value < __b3) return __n + 2;
if (__value < __b4) return __n + 3;
__value /= __b4;
__n += 4;
}
}
template<typename _Tp>
void
__to_chars_10_impl(char* __first, unsigned __len, _Tp __val) noexcept
{
static_assert(__integer_to_chars_is_unsigned<_Tp>, "implementation bug");
constexpr char __digits[201] =
"0001020304050607080910111213141516171819"
"2021222324252627282930313233343536373839"
"4041424344454647484950515253545556575859"
"6061626364656667686970717273747576777879"
"8081828384858687888990919293949596979899";
unsigned __pos = __len - 1;
while (__val >= 100)
{
auto const __num = (__val % 100) * 2;
__val /= 100;
__first[__pos] = __digits[__num + 1];
__first[__pos - 1] = __digits[__num];
__pos -= 2;
}
if (__val >= 10)
{
auto const __num = __val * 2;
__first[1] = __digits[__num + 1];
__first[0] = __digits[__num];
}
else
__first[0] = '0' + __val;
}
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __cxx11 {
inline int
stoi(const string& __str, size_t* __idx = 0, int __base = 10)
{ return __gnu_cxx::__stoa<long, int>(&std::strtol, "stoi", __str.c_str(),
__idx, __base); }
inline long
stol(const string& __str, size_t* __idx = 0, int __base = 10)
{ return __gnu_cxx::__stoa(&std::strtol, "stol", __str.c_str(),
__idx, __base); }
inline unsigned long
stoul(const string& __str, size_t* __idx = 0, int __base = 10)
{ return __gnu_cxx::__stoa(&std::strtoul, "stoul", __str.c_str(),
__idx, __base); }
inline long long
stoll(const string& __str, size_t* __idx = 0, int __base = 10)
{ return __gnu_cxx::__stoa(&std::strtoll, "stoll", __str.c_str(),
__idx, __base); }
inline unsigned long long
stoull(const string& __str, size_t* __idx = 0, int __base = 10)
{ return __gnu_cxx::__stoa(&std::strtoull, "stoull", __str.c_str(),
__idx, __base); }
inline float
stof(const string& __str, size_t* __idx = 0)
{ return __gnu_cxx::__stoa(&std::strtof, "stof", __str.c_str(), __idx); }
inline double
stod(const string& __str, size_t* __idx = 0)
{ return __gnu_cxx::__stoa(&std::strtod, "stod", __str.c_str(), __idx); }
inline long double
stold(const string& __str, size_t* __idx = 0)
{ return __gnu_cxx::__stoa(&std::strtold, "stold", __str.c_str(), __idx); }
[[__nodiscard__]]
inline string
to_string(int __val)
noexcept
{
const bool __neg = __val < 0;
const unsigned __uval = __neg ? (unsigned)~__val + 1u : __val;
const auto __len = __detail::__to_chars_len(__uval);
string __str(__neg + __len, '-');
__detail::__to_chars_10_impl(&__str[__neg], __len, __uval);
return __str;
}
[[__nodiscard__]]
inline string
to_string(unsigned __val)
noexcept
{
string __str(__detail::__to_chars_len(__val), '\0');
__detail::__to_chars_10_impl(&__str[0], __str.size(), __val);
return __str;
}
[[__nodiscard__]]
inline string
to_string(long __val)
noexcept
{
const bool __neg = __val < 0;
const unsigned long __uval = __neg ? (unsigned long)~__val + 1ul : __val;
const auto __len = __detail::__to_chars_len(__uval);
string __str(__neg + __len, '-');
__detail::__to_chars_10_impl(&__str[__neg], __len, __uval);
return __str;
}
[[__nodiscard__]]
inline string
to_string(unsigned long __val)
noexcept
{
string __str(__detail::__to_chars_len(__val), '\0');
__detail::__to_chars_10_impl(&__str[0], __str.size(), __val);
return __str;
}
[[__nodiscard__]]
inline string
to_string(long long __val)
{
const bool __neg = __val < 0;
const unsigned long long __uval
= __neg ? (unsigned long long)~__val + 1ull : __val;
const auto __len = __detail::__to_chars_len(__uval);
string __str(__neg + __len, '-');
__detail::__to_chars_10_impl(&__str[__neg], __len, __uval);
return __str;
}
[[__nodiscard__]]
inline string
to_string(unsigned long long __val)
{
string __str(__detail::__to_chars_len(__val), '\0');
__detail::__to_chars_10_impl(&__str[0], __str.size(), __val);
return __str;
}
[[__nodiscard__]]
inline string
to_string(float __val)
{
const int __n =
__gnu_cxx::__numeric_traits<float>::__max_exponent10 + 20;
return __gnu_cxx::__to_xstring<string>(&std::vsnprintf, __n,
"%f", __val);
}
[[__nodiscard__]]
inline string
to_string(double __val)
{
const int __n =
__gnu_cxx::__numeric_traits<double>::__max_exponent10 + 20;
return __gnu_cxx::__to_xstring<string>(&std::vsnprintf, __n,
"%f", __val);
}
[[__nodiscard__]]
inline string
to_string(long double __val)
{
const int __n =
__gnu_cxx::__numeric_traits<long double>::__max_exponent10 + 20;
return __gnu_cxx::__to_xstring<string>(&std::vsnprintf, __n,
"%Lf", __val);
}
inline int
stoi(const wstring& __str, size_t* __idx = 0, int __base = 10)
{ return __gnu_cxx::__stoa<long, int>(&std::wcstol, "stoi", __str.c_str(),
__idx, __base); }
inline long
stol(const wstring& __str, size_t* __idx = 0, int __base = 10)
{ return __gnu_cxx::__stoa(&std::wcstol, "stol", __str.c_str(),
__idx, __base); }
inline unsigned long
stoul(const wstring& __str, size_t* __idx = 0, int __base = 10)
{ return __gnu_cxx::__stoa(&std::wcstoul, "stoul", __str.c_str(),
__idx, __base); }
inline long long
stoll(const wstring& __str, size_t* __idx = 0, int __base = 10)
{ return __gnu_cxx::__stoa(&std::wcstoll, "stoll", __str.c_str(),
__idx, __base); }
inline unsigned long long
stoull(const wstring& __str, size_t* __idx = 0, int __base = 10)
{ return __gnu_cxx::__stoa(&std::wcstoull, "stoull", __str.c_str(),
__idx, __base); }
inline float
stof(const wstring& __str, size_t* __idx = 0)
{ return __gnu_cxx::__stoa(&std::wcstof, "stof", __str.c_str(), __idx); }
inline double
stod(const wstring& __str, size_t* __idx = 0)
{ return __gnu_cxx::__stoa(&std::wcstod, "stod", __str.c_str(), __idx); }
inline long double
stold(const wstring& __str, size_t* __idx = 0)
{ return __gnu_cxx::__stoa(&std::wcstold, "stold", __str.c_str(), __idx); }
[[__nodiscard__]]
inline wstring
to_wstring(int __val)
{ return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf, 4 * sizeof(int),
L"%d", __val); }
[[__nodiscard__]]
inline wstring
to_wstring(unsigned __val)
{ return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf,
4 * sizeof(unsigned),
L"%u", __val); }
[[__nodiscard__]]
inline wstring
to_wstring(long __val)
{ return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf, 4 * sizeof(long),
L"%ld", __val); }
[[__nodiscard__]]
inline wstring
to_wstring(unsigned long __val)
{ return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf,
4 * sizeof(unsigned long),
L"%lu", __val); }
[[__nodiscard__]]
inline wstring
to_wstring(long long __val)
{ return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf,
4 * sizeof(long long),
L"%lld", __val); }
[[__nodiscard__]]
inline wstring
to_wstring(unsigned long long __val)
{ return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf,
4 * sizeof(unsigned long long),
L"%llu", __val); }
[[__nodiscard__]]
inline wstring
to_wstring(float __val)
{
const int __n =
__gnu_cxx::__numeric_traits<float>::__max_exponent10 + 20;
return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf, __n,
L"%f", __val);
}
[[__nodiscard__]]
inline wstring
to_wstring(double __val)
{
const int __n =
__gnu_cxx::__numeric_traits<double>::__max_exponent10 + 20;
return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf, __n,
L"%f", __val);
}
[[__nodiscard__]]
inline wstring
to_wstring(long double __val)
{
const int __n =
__gnu_cxx::__numeric_traits<long double>::__max_exponent10 + 20;
return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf, __n,
L"%Lf", __val);
}
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _CharT, typename _Alloc,
typename _StrT = basic_string<_CharT, char_traits<_CharT>, _Alloc>>
struct __str_hash_base
: public __hash_base<size_t, _StrT>
{
[[__nodiscard__]]
size_t
operator()(const _StrT& __s) const noexcept
{ return _Hash_impl::hash(__s.data(), __s.length() * sizeof(_CharT)); }
};
template<typename _Alloc>
struct hash<basic_string<char, char_traits<char>, _Alloc>>
: public __str_hash_base<char, _Alloc>
{ };
template<typename _Alloc>
struct hash<basic_string<wchar_t, char_traits<wchar_t>, _Alloc>>
: public __str_hash_base<wchar_t, _Alloc>
{ };
template<typename _Alloc>
struct __is_fast_hash<hash<basic_string<wchar_t, char_traits<wchar_t>,
_Alloc>>>
: std::false_type
{ };
template<typename _Alloc>
struct hash<basic_string<char16_t, char_traits<char16_t>, _Alloc>>
: public __str_hash_base<char16_t, _Alloc>
{ };
template<typename _Alloc>
struct hash<basic_string<char32_t, char_traits<char32_t>, _Alloc>>
: public __str_hash_base<char32_t, _Alloc>
{ };
template<> struct __is_fast_hash<hash<string>> : std::false_type { };
template<> struct __is_fast_hash<hash<wstring>> : std::false_type { };
template<> struct __is_fast_hash<hash<u16string>> : std::false_type { };
template<> struct __is_fast_hash<hash<u32string>> : std::false_type { };
inline namespace literals
{
inline namespace string_literals
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wliteral-suffix"
__attribute ((__abi_tag__ ("cxx11")))
inline basic_string<char>
operator""s(const char* __str, size_t __len)
{ return basic_string<char>{__str, __len}; }
__attribute ((__abi_tag__ ("cxx11")))
inline basic_string<wchar_t>
operator""s(const wchar_t* __str, size_t __len)
{ return basic_string<wchar_t>{__str, __len}; }
__attribute ((__abi_tag__ ("cxx11")))
inline basic_string<char16_t>
operator""s(const char16_t* __str, size_t __len)
{ return basic_string<char16_t>{__str, __len}; }
__attribute ((__abi_tag__ ("cxx11")))
inline basic_string<char32_t>
operator""s(const char32_t* __str, size_t __len)
{ return basic_string<char32_t>{__str, __len}; }
#pragma GCC diagnostic pop
}
}
namespace __detail::__variant
{
template<typename> struct _Never_valueless_alt;
template<typename _Tp, typename _Traits, typename _Alloc>
struct _Never_valueless_alt<std::basic_string<_Tp, _Traits, _Alloc>>
: __and_<
is_nothrow_move_constructible<std::basic_string<_Tp, _Traits, _Alloc>>,
is_nothrow_move_assignable<std::basic_string<_Tp, _Traits, _Alloc>>
>::type
{ };
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _CharT, typename _Traits, typename _Alloc>
const typename basic_string<_CharT, _Traits, _Alloc>::size_type
basic_string<_CharT, _Traits, _Alloc>::npos;
template<typename _CharT, typename _Traits, typename _Alloc>
void
basic_string<_CharT, _Traits, _Alloc>::
swap(basic_string& __s) noexcept
{
if (this == std::__addressof(__s))
return;
_Alloc_traits::_S_on_swap(_M_get_allocator(), __s._M_get_allocator());
if (_M_is_local())
if (__s._M_is_local())
{
if (length() && __s.length())
{
_CharT __tmp_data[_S_local_capacity + 1];
traits_type::copy(__tmp_data, __s._M_local_buf,
__s.length() + 1);
traits_type::copy(__s._M_local_buf, _M_local_buf,
length() + 1);
traits_type::copy(_M_local_buf, __tmp_data,
__s.length() + 1);
}
else if (__s.length())
{
traits_type::copy(_M_local_buf, __s._M_local_buf,
__s.length() + 1);
_M_length(__s.length());
__s._M_set_length(0);
return;
}
else if (length())
{
traits_type::copy(__s._M_local_buf, _M_local_buf,
length() + 1);
__s._M_length(length());
_M_set_length(0);
return;
}
}
else
{
const size_type __tmp_capacity = __s._M_allocated_capacity;
traits_type::copy(__s._M_local_buf, _M_local_buf,
length() + 1);
_M_data(__s._M_data());
__s._M_data(__s._M_local_buf);
_M_capacity(__tmp_capacity);
}
else
{
const size_type __tmp_capacity = _M_allocated_capacity;
if (__s._M_is_local())
{
traits_type::copy(_M_local_buf, __s._M_local_buf,
__s.length() + 1);
__s._M_data(_M_data());
_M_data(_M_local_buf);
}
else
{
pointer __tmp_ptr = _M_data();
_M_data(__s._M_data());
__s._M_data(__tmp_ptr);
_M_capacity(__s._M_allocated_capacity);
}
__s._M_capacity(__tmp_capacity);
}
const size_type __tmp_length = length();
_M_length(__s.length());
__s._M_length(__tmp_length);
}
template<typename _CharT, typename _Traits, typename _Alloc>
typename basic_string<_CharT, _Traits, _Alloc>::pointer
basic_string<_CharT, _Traits, _Alloc>::
_M_create(size_type& __capacity, size_type __old_capacity)
{
if (__capacity > max_size())
std::__throw_length_error(("basic_string::_M_create"));
if (__capacity > __old_capacity && __capacity < 2 * __old_capacity)
{
__capacity = 2 * __old_capacity;
if (__capacity > max_size())
__capacity = max_size();
}
return _S_allocate(_M_get_allocator(), __capacity + 1);
}
template<typename _CharT, typename _Traits, typename _Alloc>
template<typename _InIterator>
void
basic_string<_CharT, _Traits, _Alloc>::
_M_construct(_InIterator __beg, _InIterator __end,
std::input_iterator_tag)
{
size_type __len = 0;
size_type __capacity = size_type(_S_local_capacity);
pointer __p = _M_use_local_data();
while (__beg != __end && __len < __capacity)
{
__p[__len++] = *__beg;
++__beg;
}
struct _Guard
{
explicit _Guard(basic_string* __s) : _M_guarded(__s) { }
~_Guard() { if (_M_guarded) _M_guarded->_M_dispose(); }
basic_string* _M_guarded;
} __guard(this);
while (__beg != __end)
{
if (__len == __capacity)
{
__capacity = __len + 1;
pointer __another = _M_create(__capacity, __len);
this->_S_copy(__another, _M_data(), __len);
_M_dispose();
_M_data(__another);
_M_capacity(__capacity);
}
traits_type::assign(_M_data()[__len++], *__beg);
++__beg;
}
__guard._M_guarded = 0;
_M_set_length(__len);
}
template<typename _CharT, typename _Traits, typename _Alloc>
template<typename _InIterator>
void
basic_string<_CharT, _Traits, _Alloc>::
_M_construct(_InIterator __beg, _InIterator __end,
std::forward_iterator_tag)
{
size_type __dnew = static_cast<size_type>(std::distance(__beg, __end));
if (__dnew > size_type(_S_local_capacity))
{
_M_data(_M_create(__dnew, size_type(0)));
_M_capacity(__dnew);
}
else
_M_use_local_data();
struct _Guard
{
explicit _Guard(basic_string* __s) : _M_guarded(__s) { }
~_Guard() { if (_M_guarded) _M_guarded->_M_dispose(); }
basic_string* _M_guarded;
} __guard(this);
this->_S_copy_chars(_M_data(), __beg, __end);
__guard._M_guarded = 0;
_M_set_length(__dnew);
}
template<typename _CharT, typename _Traits, typename _Alloc>
void
basic_string<_CharT, _Traits, _Alloc>::
_M_construct(size_type __n, _CharT __c)
{
if (__n > size_type(_S_local_capacity))
{
_M_data(_M_create(__n, size_type(0)));
_M_capacity(__n);
}
else
_M_use_local_data();
if (__n)
this->_S_assign(_M_data(), __n, __c);
_M_set_length(__n);
}
template<typename _CharT, typename _Traits, typename _Alloc>
void
basic_string<_CharT, _Traits, _Alloc>::
_M_assign(const basic_string& __str)
{
if (this != std::__addressof(__str))
{
const size_type __rsize = __str.length();
const size_type __capacity = capacity();
if (__rsize > __capacity)
{
size_type __new_capacity = __rsize;
pointer __tmp = _M_create(__new_capacity, __capacity);
_M_dispose();
_M_data(__tmp);
_M_capacity(__new_capacity);
}
if (__rsize)
this->_S_copy(_M_data(), __str._M_data(), __rsize);
_M_set_length(__rsize);
}
}
template<typename _CharT, typename _Traits, typename _Alloc>
void
basic_string<_CharT, _Traits, _Alloc>::
reserve(size_type __res)
{
const size_type __capacity = capacity();
if (__res <= __capacity)
return;
pointer __tmp = _M_create(__res, __capacity);
this->_S_copy(__tmp, _M_data(), length() + 1);
_M_dispose();
_M_data(__tmp);
_M_capacity(__res);
}
template<typename _CharT, typename _Traits, typename _Alloc>
void
basic_string<_CharT, _Traits, _Alloc>::
_M_mutate(size_type __pos, size_type __len1, const _CharT* __s,
size_type __len2)
{
const size_type __how_much = length() - __pos - __len1;
size_type __new_capacity = length() + __len2 - __len1;
pointer __r = _M_create(__new_capacity, capacity());
if (__pos)
this->_S_copy(__r, _M_data(), __pos);
if (__s && __len2)
this->_S_copy(__r + __pos, __s, __len2);
if (__how_much)
this->_S_copy(__r + __pos + __len2,
_M_data() + __pos + __len1, __how_much);
_M_dispose();
_M_data(__r);
_M_capacity(__new_capacity);
}
template<typename _CharT, typename _Traits, typename _Alloc>
void
basic_string<_CharT, _Traits, _Alloc>::
_M_erase(size_type __pos, size_type __n)
{
const size_type __how_much = length() - __pos - __n;
if (__how_much && __n)
this->_S_move(_M_data() + __pos, _M_data() + __pos + __n, __how_much);
_M_set_length(length() - __n);
}
template<typename _CharT, typename _Traits, typename _Alloc>
void
basic_string<_CharT, _Traits, _Alloc>::
reserve()
{
if (_M_is_local())
return;
const size_type __length = length();
const size_type __capacity = _M_allocated_capacity;
if (__length <= size_type(_S_local_capacity))
{
this->_S_copy(_M_use_local_data(), _M_data(), __length + 1);
_M_destroy(__capacity);
_M_data(_M_local_data());
}
else if (__length < __capacity)
try
{
pointer __tmp = _S_allocate(_M_get_allocator(), __length + 1);
this->_S_copy(__tmp, _M_data(), __length + 1);
_M_dispose();
_M_data(__tmp);
_M_capacity(__length);
}
catch (const __cxxabiv1::__forced_unwind&)
{ throw; }
catch (...)
{ }
}
template<typename _CharT, typename _Traits, typename _Alloc>
void
basic_string<_CharT, _Traits, _Alloc>::
resize(size_type __n, _CharT __c)
{
const size_type __size = this->size();
if (__size < __n)
this->append(__n - __size, __c);
else if (__n < __size)
this->_M_set_length(__n);
}
template<typename _CharT, typename _Traits, typename _Alloc>
basic_string<_CharT, _Traits, _Alloc>&
basic_string<_CharT, _Traits, _Alloc>::
_M_append(const _CharT* __s, size_type __n)
{
const size_type __len = __n + this->size();
if (__len <= this->capacity())
{
if (__n)
this->_S_copy(this->_M_data() + this->size(), __s, __n);
}
else
this->_M_mutate(this->size(), size_type(0), __s, __n);
this->_M_set_length(__len);
return *this;
}
template<typename _CharT, typename _Traits, typename _Alloc>
template<typename _InputIterator>
basic_string<_CharT, _Traits, _Alloc>&
basic_string<_CharT, _Traits, _Alloc>::
_M_replace_dispatch(const_iterator __i1, const_iterator __i2,
_InputIterator __k1, _InputIterator __k2,
std::__false_type)
{
const basic_string __s(__k1, __k2, this->get_allocator());
const size_type __n1 = __i2 - __i1;
return _M_replace(__i1 - begin(), __n1, __s._M_data(),
__s.size());
}
template<typename _CharT, typename _Traits, typename _Alloc>
basic_string<_CharT, _Traits, _Alloc>&
basic_string<_CharT, _Traits, _Alloc>::
_M_replace_aux(size_type __pos1, size_type __n1, size_type __n2,
_CharT __c)
{
_M_check_length(__n1, __n2, "basic_string::_M_replace_aux");
const size_type __old_size = this->size();
const size_type __new_size = __old_size + __n2 - __n1;
if (__new_size <= this->capacity())
{
pointer __p = this->_M_data() + __pos1;
const size_type __how_much = __old_size - __pos1 - __n1;
if (__how_much && __n1 != __n2)
this->_S_move(__p + __n2, __p + __n1, __how_much);
}
else
this->_M_mutate(__pos1, __n1, 0, __n2);
if (__n2)
this->_S_assign(this->_M_data() + __pos1, __n2, __c);
this->_M_set_length(__new_size);
return *this;
}
template<typename _CharT, typename _Traits, typename _Alloc>
__attribute__((__noinline__, __noclone__, __cold__)) void
basic_string<_CharT, _Traits, _Alloc>::
_M_replace_cold(pointer __p, size_type __len1, const _CharT* __s,
const size_type __len2, const size_type __how_much)
{
if (__len2 && __len2 <= __len1)
this->_S_move(__p, __s, __len2);
if (__how_much && __len1 != __len2)
this->_S_move(__p + __len2, __p + __len1, __how_much);
if (__len2 > __len1)
{
if (__s + __len2 <= __p + __len1)
this->_S_move(__p, __s, __len2);
else if (__s >= __p + __len1)
{
const size_type __poff = (__s - __p) + (__len2 - __len1);
this->_S_copy(__p, __p + __poff, __len2);
}
else
{
const size_type __nleft = (__p + __len1) - __s;
this->_S_move(__p, __s, __nleft);
this->_S_copy(__p + __nleft, __p + __len2, __len2 - __nleft);
}
}
}
template<typename _CharT, typename _Traits, typename _Alloc>
basic_string<_CharT, _Traits, _Alloc>&
basic_string<_CharT, _Traits, _Alloc>::
_M_replace(size_type __pos, size_type __len1, const _CharT* __s,
const size_type __len2)
{
_M_check_length(__len1, __len2, "basic_string::_M_replace");
const size_type __old_size = this->size();
const size_type __new_size = __old_size + __len2 - __len1;
if (__new_size <= this->capacity())
{
pointer __p = this->_M_data() + __pos;
const size_type __how_much = __old_size - __pos - __len1;
if (__builtin_expect(_M_disjunct(__s), true))
{
if (__how_much && __len1 != __len2)
this->_S_move(__p + __len2, __p + __len1, __how_much);
if (__len2)
this->_S_copy(__p, __s, __len2);
}
else
_M_replace_cold(__p, __len1, __s, __len2, __how_much);
}
else
this->_M_mutate(__pos, __len1, __s, __len2);
this->_M_set_length(__new_size);
return *this;
}
template<typename _CharT, typename _Traits, typename _Alloc>
typename basic_string<_CharT, _Traits, _Alloc>::size_type
basic_string<_CharT, _Traits, _Alloc>::
copy(_CharT* __s, size_type __n, size_type __pos) const
{
_M_check(__pos, "basic_string::copy");
__n = _M_limit(__pos, __n);
;
if (__n)
_S_copy(__s, _M_data() + __pos, __n);
return __n;
}
template<typename _CharT, typename _Traits, typename _Alloc>
typename basic_string<_CharT, _Traits, _Alloc>::size_type
basic_string<_CharT, _Traits, _Alloc>::
find(const _CharT* __s, size_type __pos, size_type __n) const
noexcept
{
;
const size_type __size = this->size();
if (__n == 0)
return __pos <= __size ? __pos : npos;
if (__pos >= __size)
return npos;
const _CharT __elem0 = __s[0];
const _CharT* const __data = data();
const _CharT* __first = __data + __pos;
const _CharT* const __last = __data + __size;
size_type __len = __size - __pos;
while (__len >= __n)
{
__first = traits_type::find(__first, __len - __n + 1, __elem0);
if (!__first)
return npos;
if (traits_type::compare(__first, __s, __n) == 0)
return __first - __data;
__len = __last - ++__first;
}
return npos;
}
template<typename _CharT, typename _Traits, typename _Alloc>
typename basic_string<_CharT, _Traits, _Alloc>::size_type
basic_string<_CharT, _Traits, _Alloc>::
find(_CharT __c, size_type __pos) const noexcept
{
size_type __ret = npos;
const size_type __size = this->size();
if (__pos < __size)
{
const _CharT* __data = _M_data();
const size_type __n = __size - __pos;
const _CharT* __p = traits_type::find(__data + __pos, __n, __c);
if (__p)
__ret = __p - __data;
}
return __ret;
}
template<typename _CharT, typename _Traits, typename _Alloc>
typename basic_string<_CharT, _Traits, _Alloc>::size_type
basic_string<_CharT, _Traits, _Alloc>::
rfind(const _CharT* __s, size_type __pos, size_type __n) const
noexcept
{
;
const size_type __size = this->size();
if (__n <= __size)
{
__pos = std::min(size_type(__size - __n), __pos);
const _CharT* __data = _M_data();
do
{
if (traits_type::compare(__data + __pos, __s, __n) == 0)
return __pos;
}
while (__pos-- > 0);
}
return npos;
}
template<typename _CharT, typename _Traits, typename _Alloc>
typename basic_string<_CharT, _Traits, _Alloc>::size_type
basic_string<_CharT, _Traits, _Alloc>::
rfind(_CharT __c, size_type __pos) const noexcept
{
size_type __size = this->size();
if (__size)
{
if (--__size > __pos)
__size = __pos;
for (++__size; __size-- > 0; )
if (traits_type::eq(_M_data()[__size], __c))
return __size;
}
return npos;
}
template<typename _CharT, typename _Traits, typename _Alloc>
typename basic_string<_CharT, _Traits, _Alloc>::size_type
basic_string<_CharT, _Traits, _Alloc>::
find_first_of(const _CharT* __s, size_type __pos, size_type __n) const
noexcept
{
;
for (; __n && __pos < this->size(); ++__pos)
{
const _CharT* __p = traits_type::find(__s, __n, _M_data()[__pos]);
if (__p)
return __pos;
}
return npos;
}
template<typename _CharT, typename _Traits, typename _Alloc>
typename basic_string<_CharT, _Traits, _Alloc>::size_type
basic_string<_CharT, _Traits, _Alloc>::
find_last_of(const _CharT* __s, size_type __pos, size_type __n) const
noexcept
{
;
size_type __size = this->size();
if (__size && __n)
{
if (--__size > __pos)
__size = __pos;
do
{
if (traits_type::find(__s, __n, _M_data()[__size]))
return __size;
}
while (__size-- != 0);
}
return npos;
}
template<typename _CharT, typename _Traits, typename _Alloc>
typename basic_string<_CharT, _Traits, _Alloc>::size_type
basic_string<_CharT, _Traits, _Alloc>::
find_first_not_of(const _CharT* __s, size_type __pos, size_type __n) const
noexcept
{
;
for (; __pos < this->size(); ++__pos)
if (!traits_type::find(__s, __n, _M_data()[__pos]))
return __pos;
return npos;
}
template<typename _CharT, typename _Traits, typename _Alloc>
typename basic_string<_CharT, _Traits, _Alloc>::size_type
basic_string<_CharT, _Traits, _Alloc>::
find_first_not_of(_CharT __c, size_type __pos) const noexcept
{
for (; __pos < this->size(); ++__pos)
if (!traits_type::eq(_M_data()[__pos], __c))
return __pos;
return npos;
}
template<typename _CharT, typename _Traits, typename _Alloc>
typename basic_string<_CharT, _Traits, _Alloc>::size_type
basic_string<_CharT, _Traits, _Alloc>::
find_last_not_of(const _CharT* __s, size_type __pos, size_type __n) const
noexcept
{
;
size_type __size = this->size();
if (__size)
{
if (--__size > __pos)
__size = __pos;
do
{
if (!traits_type::find(__s, __n, _M_data()[__size]))
return __size;
}
while (__size--);
}
return npos;
}
template<typename _CharT, typename _Traits, typename _Alloc>
typename basic_string<_CharT, _Traits, _Alloc>::size_type
basic_string<_CharT, _Traits, _Alloc>::
find_last_not_of(_CharT __c, size_type __pos) const noexcept
{
size_type __size = this->size();
if (__size)
{
if (--__size > __pos)
__size = __pos;
do
{
if (!traits_type::eq(_M_data()[__size], __c))
return __size;
}
while (__size--);
}
return npos;
}
template<typename _CharT, typename _Traits, typename _Alloc>
basic_istream<_CharT, _Traits>&
operator>>(basic_istream<_CharT, _Traits>& __in,
basic_string<_CharT, _Traits, _Alloc>& __str)
{
typedef basic_istream<_CharT, _Traits> __istream_type;
typedef basic_string<_CharT, _Traits, _Alloc> __string_type;
typedef typename __istream_type::ios_base __ios_base;
typedef typename __istream_type::int_type __int_type;
typedef typename __string_type::size_type __size_type;
typedef ctype<_CharT> __ctype_type;
typedef typename __ctype_type::ctype_base __ctype_base;
__size_type __extracted = 0;
typename __ios_base::iostate __err = __ios_base::goodbit;
typename __istream_type::sentry __cerb(__in, false);
if (__cerb)
{
try
{
__str.erase();
_CharT __buf[128];
__size_type __len = 0;
const streamsize __w = __in.width();
const __size_type __n = __w > 0 ? static_cast<__size_type>(__w)
: __str.max_size();
const __ctype_type& __ct = use_facet<__ctype_type>(__in.getloc());
const __int_type __eof = _Traits::eof();
__int_type __c = __in.rdbuf()->sgetc();
while (__extracted < __n
&& !_Traits::eq_int_type(__c, __eof)
&& !__ct.is(__ctype_base::space,
_Traits::to_char_type(__c)))
{
if (__len == sizeof(__buf) / sizeof(_CharT))
{
__str.append(__buf, sizeof(__buf) / sizeof(_CharT));
__len = 0;
}
__buf[__len++] = _Traits::to_char_type(__c);
++__extracted;
__c = __in.rdbuf()->snextc();
}
__str.append(__buf, __len);
if (__extracted < __n && _Traits::eq_int_type(__c, __eof))
__err |= __ios_base::eofbit;
__in.width(0);
}
catch(__cxxabiv1::__forced_unwind&)
{
__in._M_setstate(__ios_base::badbit);
throw;
}
catch(...)
{
__in._M_setstate(__ios_base::badbit);
}
}
if (!__extracted)
__err |= __ios_base::failbit;
if (__err)
__in.setstate(__err);
return __in;
}
template<typename _CharT, typename _Traits, typename _Alloc>
basic_istream<_CharT, _Traits>&
getline(basic_istream<_CharT, _Traits>& __in,
basic_string<_CharT, _Traits, _Alloc>& __str, _CharT __delim)
{
typedef basic_istream<_CharT, _Traits> __istream_type;
typedef basic_string<_CharT, _Traits, _Alloc> __string_type;
typedef typename __istream_type::ios_base __ios_base;
typedef typename __istream_type::int_type __int_type;
typedef typename __string_type::size_type __size_type;
__size_type __extracted = 0;
const __size_type __n = __str.max_size();
typename __ios_base::iostate __err = __ios_base::goodbit;
typename __istream_type::sentry __cerb(__in, true);
if (__cerb)
{
try
{
__str.erase();
const __int_type __idelim = _Traits::to_int_type(__delim);
const __int_type __eof = _Traits::eof();
__int_type __c = __in.rdbuf()->sgetc();
while (__extracted < __n
&& !_Traits::eq_int_type(__c, __eof)
&& !_Traits::eq_int_type(__c, __idelim))
{
__str += _Traits::to_char_type(__c);
++__extracted;
__c = __in.rdbuf()->snextc();
}
if (_Traits::eq_int_type(__c, __eof))
__err |= __ios_base::eofbit;
else if (_Traits::eq_int_type(__c, __idelim))
{
++__extracted;
__in.rdbuf()->sbumpc();
}
else
__err |= __ios_base::failbit;
}
catch(__cxxabiv1::__forced_unwind&)
{
__in._M_setstate(__ios_base::badbit);
throw;
}
catch(...)
{
__in._M_setstate(__ios_base::badbit);
}
}
if (!__extracted)
__err |= __ios_base::failbit;
if (__err)
__in.setstate(__err);
return __in;
}
extern template class basic_string<char>;
extern template
basic_istream<char>&
operator>>(basic_istream<char>&, string&);
extern template
basic_ostream<char>&
operator<<(basic_ostream<char>&, const string&);
extern template
basic_istream<char>&
getline(basic_istream<char>&, string&, char);
extern template
basic_istream<char>&
getline(basic_istream<char>&, string&);
extern template class basic_string<wchar_t>;
extern template
basic_istream<wchar_t>&
operator>>(basic_istream<wchar_t>&, wstring&);
extern template
basic_ostream<wchar_t>&
operator<<(basic_ostream<wchar_t>&, const wstring&);
extern template
basic_istream<wchar_t>&
getline(basic_istream<wchar_t>&, wstring&, wchar_t);
extern template
basic_istream<wchar_t>&
getline(basic_istream<wchar_t>&, wstring&);
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace pmr {
template<typename _CharT, typename _Traits = char_traits<_CharT>>
using basic_string = std::basic_string<_CharT, _Traits,
polymorphic_allocator<_CharT>>;
using string = basic_string<char>;
using u16string = basic_string<char16_t>;
using u32string = basic_string<char32_t>;
using wstring = basic_string<wchar_t>;
}
}
namespace std
{
using ::int8_t;
using ::int16_t;
using ::int32_t;
using ::int64_t;
using ::int_fast8_t;
using ::int_fast16_t;
using ::int_fast32_t;
using ::int_fast64_t;
using ::int_least8_t;
using ::int_least16_t;
using ::int_least32_t;
using ::int_least64_t;
using ::intmax_t;
using ::intptr_t;
using ::uint8_t;
using ::uint16_t;
using ::uint32_t;
using ::uint64_t;
using ::uint_fast8_t;
using ::uint_fast16_t;
using ::uint_fast32_t;
using ::uint_fast64_t;
using ::uint_least8_t;
using ::uint_least16_t;
using ::uint_least32_t;
using ::uint_least64_t;
using ::uintmax_t;
using ::uintptr_t;
}
namespace std __attribute__ ((__visibility__ ("default")))
{
class locale
{
public:
typedef int category;
class facet;
class id;
class _Impl;
friend class facet;
friend class _Impl;
template<typename _Facet>
friend bool
has_facet(const locale&) throw();
template<typename _Facet>
friend const _Facet&
use_facet(const locale&);
template<typename _Facet>
friend const _Facet*
__try_use_facet(const locale&) noexcept;
template<typename _Cache>
friend struct __use_cache;
static const category none = 0;
static const category ctype = 1L << 0;
static const category numeric = 1L << 1;
static const category collate = 1L << 2;
static const category time = 1L << 3;
static const category monetary = 1L << 4;
static const category messages = 1L << 5;
static const category all = (ctype | numeric | collate |
time | monetary | messages);
locale() throw();
locale(const locale& __other) throw();
explicit
locale(const char* __s);
locale(const locale& __base, const char* __s, category __cat);
explicit
locale(const std::string& __s) : locale(__s.c_str()) { }
locale(const locale& __base, const std::string& __s, category __cat)
: locale(__base, __s.c_str(), __cat) { }
locale(const locale& __base, const locale& __add, category __cat);
template<typename _Facet>
locale(const locale& __other, _Facet* __f);
~locale() throw();
const locale&
operator=(const locale& __other) throw();
template<typename _Facet>
locale
combine(const locale& __other) const;
__attribute ((__abi_tag__ ("cxx11")))
string
name() const;
bool
operator==(const locale& __other) const throw();
bool
operator!=(const locale& __other) const throw()
{ return !(this->operator==(__other)); }
template<typename _Char, typename _Traits, typename _Alloc>
bool
operator()(const basic_string<_Char, _Traits, _Alloc>& __s1,
const basic_string<_Char, _Traits, _Alloc>& __s2) const;
static locale
global(const locale& __loc);
static const locale&
classic();
private:
_Impl* _M_impl;
static _Impl* _S_classic;
static _Impl* _S_global;
static const char* const* const _S_categories;
enum { _S_categories_size = 6 + 6 };
static __gthread_once_t _S_once;
explicit
locale(_Impl*) throw();
static void
_S_initialize();
static void
_S_initialize_once() throw();
static category
_S_normalize_category(category);
void
_M_coalesce(const locale& __base, const locale& __add, category __cat);
static const id* const _S_twinned_facets[];
};
class locale::facet
{
private:
friend class locale;
friend class locale::_Impl;
mutable _Atomic_word _M_refcount;
static __c_locale _S_c_locale;
static const char _S_c_name[2];
static __gthread_once_t _S_once;
static void
_S_initialize_once();
protected:
explicit
facet(size_t __refs = 0) throw() : _M_refcount(__refs ? 1 : 0)
{ }
virtual
~facet();
static void
_S_create_c_locale(__c_locale& __cloc, const char* __s,
__c_locale __old = 0);
static __c_locale
_S_clone_c_locale(__c_locale& __cloc) throw();
static void
_S_destroy_c_locale(__c_locale& __cloc);
static __c_locale
_S_lc_ctype_c_locale(__c_locale __cloc, const char* __s);
static __c_locale
_S_get_c_locale();
__attribute__ ((__const__)) static const char*
_S_get_c_name() throw();
facet(const facet&) = delete;
facet&
operator=(const facet&) = delete;
private:
void
_M_add_reference() const throw()
{ __gnu_cxx::__atomic_add_dispatch(&_M_refcount, 1); }
void
_M_remove_reference() const throw()
{
;
if (__gnu_cxx::__exchange_and_add_dispatch(&_M_refcount, -1) == 1)
{
;
try
{ delete this; }
catch(...)
{ }
}
}
const facet* _M_sso_shim(const id*) const;
const facet* _M_cow_shim(const id*) const;
protected:
class __shim;
};
class locale::id
{
private:
friend class locale;
friend class locale::_Impl;
template<typename _Facet>
friend const _Facet&
use_facet(const locale&);
template<typename _Facet>
friend bool
has_facet(const locale&) throw();
template<typename _Facet>
friend const _Facet*
__try_use_facet(const locale&) noexcept;
mutable size_t _M_index;
static _Atomic_word _S_refcount;
void
operator=(const id&);
id(const id&);
public:
id() { }
size_t
_M_id() const throw();
};
class locale::_Impl
{
public:
friend class locale;
friend class locale::facet;
template<typename _Facet>
friend bool
has_facet(const locale&) throw();
template<typename _Facet>
friend const _Facet&
use_facet(const locale&);
template<typename _Facet>
friend const _Facet*
__try_use_facet(const locale&) noexcept;
template<typename _Cache>
friend struct __use_cache;
private:
_Atomic_word _M_refcount;
const facet** _M_facets;
size_t _M_facets_size;
const facet** _M_caches;
char** _M_names;
static const locale::id* const _S_id_ctype[];
static const locale::id* const _S_id_numeric[];
static const locale::id* const _S_id_collate[];
static const locale::id* const _S_id_time[];
static const locale::id* const _S_id_monetary[];
static const locale::id* const _S_id_messages[];
static const locale::id* const* const _S_facet_categories[];
void
_M_add_reference() throw()
{ __gnu_cxx::__atomic_add_dispatch(&_M_refcount, 1); }
void
_M_remove_reference() throw()
{
;
if (__gnu_cxx::__exchange_and_add_dispatch(&_M_refcount, -1) == 1)
{
;
try
{ delete this; }
catch(...)
{ }
}
}
_Impl(const _Impl&, size_t);
_Impl(const char*, size_t);
_Impl(size_t) throw();
~_Impl() throw();
_Impl(const _Impl&);
void
operator=(const _Impl&);
bool
_M_check_same_name()
{
bool __ret = true;
if (_M_names[1])
for (size_t __i = 0; __ret && __i < _S_categories_size - 1; ++__i)
__ret = __builtin_strcmp(_M_names[__i], _M_names[__i + 1]) == 0;
return __ret;
}
void
_M_replace_categories(const _Impl*, category);
void
_M_replace_category(const _Impl*, const locale::id* const*);
void
_M_replace_facet(const _Impl*, const locale::id*);
void
_M_install_facet(const locale::id*, const facet*);
template<typename _Facet>
void
_M_init_facet(_Facet* __facet)
{ _M_install_facet(&_Facet::id, __facet); }
template<typename _Facet>
void
_M_init_facet_unchecked(_Facet* __facet)
{
__facet->_M_add_reference();
_M_facets[_Facet::id._M_id()] = __facet;
}
void
_M_install_cache(const facet*, size_t);
void _M_init_extra(facet**);
void _M_init_extra(void*, void*, const char*, const char*);
};
template<typename _CharT>
class __cxx11:: collate : public locale::facet
{
public:
typedef _CharT char_type;
typedef basic_string<_CharT> string_type;
protected:
__c_locale _M_c_locale_collate;
public:
static locale::id id;
explicit
collate(size_t __refs = 0)
: facet(__refs), _M_c_locale_collate(_S_get_c_locale())
{ }
explicit
collate(__c_locale __cloc, size_t __refs = 0)
: facet(__refs), _M_c_locale_collate(_S_clone_c_locale(__cloc))
{ }
int
compare(const _CharT* __lo1, const _CharT* __hi1,
const _CharT* __lo2, const _CharT* __hi2) const
{ return this->do_compare(__lo1, __hi1, __lo2, __hi2); }
string_type
transform(const _CharT* __lo, const _CharT* __hi) const
{ return this->do_transform(__lo, __hi); }
long
hash(const _CharT* __lo, const _CharT* __hi) const
{ return this->do_hash(__lo, __hi); }
int
_M_compare(const _CharT*, const _CharT*) const throw();
size_t
_M_transform(_CharT*, const _CharT*, size_t) const throw();
protected:
virtual
~collate()
{ _S_destroy_c_locale(_M_c_locale_collate); }
virtual int
do_compare(const _CharT* __lo1, const _CharT* __hi1,
const _CharT* __lo2, const _CharT* __hi2) const;
virtual string_type
do_transform(const _CharT* __lo, const _CharT* __hi) const;
virtual long
do_hash(const _CharT* __lo, const _CharT* __hi) const;
};
template<typename _CharT>
locale::id collate<_CharT>::id;
template<>
int
collate<char>::_M_compare(const char*, const char*) const throw();
template<>
size_t
collate<char>::_M_transform(char*, const char*, size_t) const throw();
template<>
int
collate<wchar_t>::_M_compare(const wchar_t*, const wchar_t*) const throw();
template<>
size_t
collate<wchar_t>::_M_transform(wchar_t*, const wchar_t*, size_t) const throw();
template<typename _CharT>
class __cxx11:: collate_byname : public collate<_CharT>
{
public:
typedef _CharT char_type;
typedef basic_string<_CharT> string_type;
explicit
collate_byname(const char* __s, size_t __refs = 0)
: collate<_CharT>(__refs)
{
if (__builtin_strcmp(__s, "C") != 0
&& __builtin_strcmp(__s, "POSIX") != 0)
{
this->_S_destroy_c_locale(this->_M_c_locale_collate);
this->_S_create_c_locale(this->_M_c_locale_collate, __s);
}
}
explicit
collate_byname(const string& __s, size_t __refs = 0)
: collate_byname(__s.c_str(), __refs) { }
protected:
virtual
~collate_byname() { }
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Facet>
locale::
locale(const locale& __other, _Facet* __f)
{
_M_impl = new _Impl(*__other._M_impl, 1);
try
{ _M_impl->_M_install_facet(&_Facet::id, __f); }
catch(...)
{
_M_impl->_M_remove_reference();
throw;
}
delete [] _M_impl->_M_names[0];
_M_impl->_M_names[0] = 0;
}
template<typename _Facet>
locale
locale::
combine(const locale& __other) const
{
_Impl* __tmp = new _Impl(*_M_impl, 1);
try
{
__tmp->_M_replace_facet(__other._M_impl, &_Facet::id);
}
catch(...)
{
__tmp->_M_remove_reference();
throw;
}
return locale(__tmp);
}
template<typename _CharT, typename _Traits, typename _Alloc>
bool
locale::
operator()(const basic_string<_CharT, _Traits, _Alloc>& __s1,
const basic_string<_CharT, _Traits, _Alloc>& __s2) const
{
typedef std::collate<_CharT> __collate_type;
const __collate_type& __collate = use_facet<__collate_type>(*this);
return (__collate.compare(__s1.data(), __s1.data() + __s1.length(),
__s2.data(), __s2.data() + __s2.length()) < 0);
}
template<typename _Facet>
inline const _Facet*
__try_use_facet(const locale& __loc) noexcept
{
const size_t __i = _Facet::id._M_id();
const locale::facet** __facets = __loc._M_impl->_M_facets;
if constexpr (__is_same(_Facet, ctype<char>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, num_get<char>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, num_put<char>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, codecvt<char, char, mbstate_t>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, collate<char>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, moneypunct<char>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, moneypunct<char, true>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, money_get<char>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, money_put<char>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, numpunct<char>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, time_get<char>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, time_put<char>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, messages<char>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, ctype<wchar_t>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, num_get<wchar_t>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, num_put<wchar_t>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, codecvt<wchar_t, char, mbstate_t>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, collate<wchar_t>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, moneypunct<wchar_t>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, moneypunct<wchar_t, true>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, money_get<wchar_t>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, money_put<wchar_t>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, numpunct<wchar_t>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, time_get<wchar_t>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, time_put<wchar_t>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, messages<wchar_t>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, codecvt<char16_t, char, mbstate_t>)) return static_cast<const _Facet*>(__facets[__i]);
if constexpr (__is_same(_Facet, codecvt<char32_t, char, mbstate_t>)) return static_cast<const _Facet*>(__facets[__i]);
if (__i >= __loc._M_impl->_M_facets_size || !__facets[__i])
return 0;
return dynamic_cast<const _Facet*>(__facets[__i]);
}
template<typename _Facet>
inline bool
has_facet(const locale& __loc) throw()
{
static_assert(__is_base_of(locale::facet, _Facet),
"template argument must be derived from locale::facet");
return std::__try_use_facet<_Facet>(__loc) != 0;
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdangling-reference"
template<typename _Facet>
inline const _Facet&
use_facet(const locale& __loc)
{
static_assert(__is_base_of(locale::facet, _Facet),
"template argument must be derived from locale::facet");
if (const _Facet* __f = std::__try_use_facet<_Facet>(__loc))
return *__f;
__throw_bad_cast();
}
#pragma GCC diagnostic pop
template<typename _CharT>
int
collate<_CharT>::_M_compare(const _CharT*, const _CharT*) const throw ()
{ return 0; }
template<typename _CharT>
size_t
collate<_CharT>::_M_transform(_CharT*, const _CharT*, size_t) const throw ()
{ return 0; }
template<typename _CharT>
int
collate<_CharT>::
do_compare(const _CharT* __lo1, const _CharT* __hi1,
const _CharT* __lo2, const _CharT* __hi2) const
{
const string_type __one(__lo1, __hi1);
const string_type __two(__lo2, __hi2);
const _CharT* __p = __one.c_str();
const _CharT* __pend = __one.data() + __one.length();
const _CharT* __q = __two.c_str();
const _CharT* __qend = __two.data() + __two.length();
for (;;)
{
const int __res = _M_compare(__p, __q);
if (__res)
return __res;
__p += char_traits<_CharT>::length(__p);
__q += char_traits<_CharT>::length(__q);
if (__p == __pend && __q == __qend)
return 0;
else if (__p == __pend)
return -1;
else if (__q == __qend)
return 1;
__p++;
__q++;
}
}
template<typename _CharT>
typename collate<_CharT>::string_type
collate<_CharT>::
do_transform(const _CharT* __lo, const _CharT* __hi) const
{
string_type __ret;
const string_type __str(__lo, __hi);
const _CharT* __p = __str.c_str();
const _CharT* __pend = __str.data() + __str.length();
size_t __len = (__hi - __lo) * 2;
_CharT* __c = new _CharT[__len];
try
{
for (;;)
{
size_t __res = _M_transform(__c, __p, __len);
if (__res >= __len)
{
__len = __res + 1;
delete [] __c, __c = 0;
__c = new _CharT[__len];
__res = _M_transform(__c, __p, __len);
}
__ret.append(__c, __res);
__p += char_traits<_CharT>::length(__p);
if (__p == __pend)
break;
__p++;
__ret.push_back(_CharT());
}
}
catch(...)
{
delete [] __c;
throw;
}
delete [] __c;
return __ret;
}
template<typename _CharT>
long
collate<_CharT>::
do_hash(const _CharT* __lo, const _CharT* __hi) const
{
unsigned long __val = 0;
for (; __lo < __hi; ++__lo)
__val =
*__lo + ((__val << 7)
| (__val >> (__gnu_cxx::__numeric_traits<unsigned long>::
__digits - 7)));
return static_cast<long>(__val);
}
extern template class collate<char>;
extern template class collate_byname<char>;
extern template
const collate<char>*
__try_use_facet<collate<char> >(const locale&) noexcept;
extern template
const collate<char>&
use_facet<collate<char> >(const locale&);
extern template
bool
has_facet<collate<char> >(const locale&);
extern template class collate<wchar_t>;
extern template class collate_byname<wchar_t>;
extern template
const collate<wchar_t>*
__try_use_facet<collate<wchar_t> >(const locale&) noexcept;
extern template
const collate<wchar_t>&
use_facet<collate<wchar_t> >(const locale&);
extern template
bool
has_facet<collate<wchar_t> >(const locale&);
}
namespace std __attribute__ ((__visibility__ ("default")))
{
enum class errc
{
address_family_not_supported = 97,
address_in_use = 98,
address_not_available = 99,
already_connected = 106,
argument_list_too_long = 7,
argument_out_of_domain = 33,
bad_address = 14,
bad_file_descriptor = 9,
bad_message = 74,
broken_pipe = 32,
connection_aborted = 103,
connection_already_in_progress = 114,
connection_refused = 111,
connection_reset = 104,
cross_device_link = 18,
destination_address_required = 89,
device_or_resource_busy = 16,
directory_not_empty = 39,
executable_format_error = 8,
file_exists = 17,
file_too_large = 27,
filename_too_long = 36,
function_not_supported = 38,
host_unreachable = 113,
identifier_removed = 43,
illegal_byte_sequence = 84,
inappropriate_io_control_operation = 25,
interrupted = 4,
invalid_argument = 22,
invalid_seek = 29,
io_error = 5,
is_a_directory = 21,
message_size = 90,
network_down = 100,
network_reset = 102,
network_unreachable = 101,
no_buffer_space = 105,
no_child_process = 10,
no_link = 67,
no_lock_available = 37,
no_message_available = 61,
no_message = 42,
no_protocol_option = 92,
no_space_on_device = 28,
no_stream_resources = 63,
no_such_device_or_address = 6,
no_such_device = 19,
no_such_file_or_directory = 2,
no_such_process = 3,
not_a_directory = 20,
not_a_socket = 88,
not_a_stream = 60,
not_connected = 107,
not_enough_memory = 12,
not_supported = 95,
operation_canceled = 125,
operation_in_progress = 115,
operation_not_permitted = 1,
operation_not_supported = 95,
operation_would_block = 11,
owner_dead = 130,
permission_denied = 13,
protocol_error = 71,
protocol_not_supported = 93,
read_only_file_system = 30,
resource_deadlock_would_occur = 35,
resource_unavailable_try_again = 11,
result_out_of_range = 34,
state_not_recoverable = 131,
stream_timeout = 62,
text_file_busy = 26,
timed_out = 110,
too_many_files_open_in_system = 23,
too_many_files_open = 24,
too_many_links = 31,
too_many_symbolic_link_levels = 40,
value_too_large = 75,
wrong_protocol_type = 91
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
struct __cow_string
{
union {
const char* _M_p;
char _M_bytes[sizeof(const char*)];
};
__cow_string();
__cow_string(const std::string&);
__cow_string(const char*, size_t);
__cow_string(const __cow_string&) noexcept;
__cow_string& operator=(const __cow_string&) noexcept;
~__cow_string();
__cow_string(__cow_string&&) noexcept;
__cow_string& operator=(__cow_string&&) noexcept;
};
typedef basic_string<char> __sso_string;
class logic_error : public exception
{
__cow_string _M_msg;
public:
explicit
logic_error(const string& __arg) ;
explicit
logic_error(const char*) ;
logic_error(logic_error&&) noexcept;
logic_error& operator=(logic_error&&) noexcept;
logic_error(const logic_error&) noexcept;
logic_error& operator=(const logic_error&) noexcept;
virtual ~logic_error() noexcept;
virtual const char*
what() const noexcept;
};
class domain_error : public logic_error
{
public:
explicit domain_error(const string& __arg) ;
explicit domain_error(const char*) ;
domain_error(const domain_error&) = default;
domain_error& operator=(const domain_error&) = default;
domain_error(domain_error&&) = default;
domain_error& operator=(domain_error&&) = default;
virtual ~domain_error() noexcept;
};
class invalid_argument : public logic_error
{
public:
explicit invalid_argument(const string& __arg) ;
explicit invalid_argument(const char*) ;
invalid_argument(const invalid_argument&) = default;
invalid_argument& operator=(const invalid_argument&) = default;
invalid_argument(invalid_argument&&) = default;
invalid_argument& operator=(invalid_argument&&) = default;
virtual ~invalid_argument() noexcept;
};
class length_error : public logic_error
{
public:
explicit length_error(const string& __arg) ;
explicit length_error(const char*) ;
length_error(const length_error&) = default;
length_error& operator=(const length_error&) = default;
length_error(length_error&&) = default;
length_error& operator=(length_error&&) = default;
virtual ~length_error() noexcept;
};
class out_of_range : public logic_error
{
public:
explicit out_of_range(const string& __arg) ;
explicit out_of_range(const char*) ;
out_of_range(const out_of_range&) = default;
out_of_range& operator=(const out_of_range&) = default;
out_of_range(out_of_range&&) = default;
out_of_range& operator=(out_of_range&&) = default;
virtual ~out_of_range() noexcept;
};
class runtime_error : public exception
{
__cow_string _M_msg;
public:
explicit
runtime_error(const string& __arg) ;
explicit
runtime_error(const char*) ;
runtime_error(runtime_error&&) noexcept;
runtime_error& operator=(runtime_error&&) noexcept;
runtime_error(const runtime_error&) noexcept;
runtime_error& operator=(const runtime_error&) noexcept;
virtual ~runtime_error() noexcept;
virtual const char*
what() const noexcept;
};
class range_error : public runtime_error
{
public:
explicit range_error(const string& __arg) ;
explicit range_error(const char*) ;
range_error(const range_error&) = default;
range_error& operator=(const range_error&) = default;
range_error(range_error&&) = default;
range_error& operator=(range_error&&) = default;
virtual ~range_error() noexcept;
};
class overflow_error : public runtime_error
{
public:
explicit overflow_error(const string& __arg) ;
explicit overflow_error(const char*) ;
overflow_error(const overflow_error&) = default;
overflow_error& operator=(const overflow_error&) = default;
overflow_error(overflow_error&&) = default;
overflow_error& operator=(overflow_error&&) = default;
virtual ~overflow_error() noexcept;
};
class underflow_error : public runtime_error
{
public:
explicit underflow_error(const string& __arg) ;
explicit underflow_error(const char*) ;
underflow_error(const underflow_error&) = default;
underflow_error& operator=(const underflow_error&) = default;
underflow_error(underflow_error&&) = default;
underflow_error& operator=(underflow_error&&) = default;
virtual ~underflow_error() noexcept;
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
class error_code;
class error_condition;
class system_error;
template<typename _Tp>
struct is_error_code_enum : public false_type { };
template<typename _Tp>
struct is_error_condition_enum : public false_type { };
template<>
struct is_error_condition_enum<errc>
: public true_type { };
template <typename _Tp>
inline constexpr bool is_error_code_enum_v =
is_error_code_enum<_Tp>::value;
template <typename _Tp>
inline constexpr bool is_error_condition_enum_v =
is_error_condition_enum<_Tp>::value;
inline namespace _V2 {
class error_category
{
public:
constexpr error_category() noexcept = default;
virtual ~error_category();
error_category(const error_category&) = delete;
error_category& operator=(const error_category&) = delete;
virtual const char*
name() const noexcept = 0;
private:
__attribute ((__abi_tag__ ("cxx11")))
virtual __cow_string
_M_message(int) const;
public:
__attribute ((__abi_tag__ ("cxx11")))
virtual string
message(int) const = 0;
public:
virtual error_condition
default_error_condition(int __i) const noexcept;
virtual bool
equivalent(int __i, const error_condition& __cond) const noexcept;
virtual bool
equivalent(const error_code& __code, int __i) const noexcept;
[[__nodiscard__]]
bool
operator==(const error_category& __other) const noexcept
{ return this == &__other; }
bool
operator<(const error_category& __other) const noexcept
{ return less<const error_category*>()(this, &__other); }
bool
operator!=(const error_category& __other) const noexcept
{ return this != &__other; }
};
[[__nodiscard__, __gnu__::__const__]]
const error_category&
generic_category() noexcept;
[[__nodiscard__, __gnu__::__const__]]
const error_category&
system_category() noexcept;
}
namespace __adl_only
{
void make_error_code() = delete;
void make_error_condition() = delete;
}
class error_code
{
template<typename _ErrorCodeEnum>
using _Check
= __enable_if_t<is_error_code_enum<_ErrorCodeEnum>::value>;
public:
error_code() noexcept
: _M_value(0), _M_cat(&system_category()) { }
error_code(int __v, const error_category& __cat) noexcept
: _M_value(__v), _M_cat(&__cat) { }
template<typename _ErrorCodeEnum,
typename = _Check<_ErrorCodeEnum>>
error_code(_ErrorCodeEnum __e) noexcept
{
using __adl_only::make_error_code;
*this = make_error_code(__e);
}
error_code(const error_code&) = default;
error_code& operator=(const error_code&) = default;
void
assign(int __v, const error_category& __cat) noexcept
{
_M_value = __v;
_M_cat = &__cat;
}
void
clear() noexcept
{ assign(0, system_category()); }
[[__nodiscard__]]
int
value() const noexcept { return _M_value; }
[[__nodiscard__]]
const error_category&
category() const noexcept { return *_M_cat; }
error_condition
default_error_condition() const noexcept;
__attribute ((__abi_tag__ ("cxx11")))
string
message() const
{ return category().message(value()); }
[[__nodiscard__]]
explicit operator bool() const noexcept
{ return _M_value != 0; }
private:
int _M_value;
const error_category* _M_cat;
};
[[__nodiscard__]]
inline error_code
make_error_code(errc __e) noexcept
{ return error_code(static_cast<int>(__e), generic_category()); }
inline bool
operator<(const error_code& __lhs, const error_code& __rhs) noexcept
{
return (__lhs.category() < __rhs.category()
|| (__lhs.category() == __rhs.category()
&& __lhs.value() < __rhs.value()));
}
template<typename _CharT, typename _Traits>
basic_ostream<_CharT, _Traits>&
operator<<(basic_ostream<_CharT, _Traits>& __os, const error_code& __e)
{ return (__os << __e.category().name() << ':' << __e.value()); }
class error_condition
{
template<typename _ErrorConditionEnum>
using _Check
= __enable_if_t<is_error_condition_enum<_ErrorConditionEnum>::value>;
public:
error_condition() noexcept
: _M_value(0), _M_cat(&generic_category()) { }
error_condition(int __v, const error_category& __cat) noexcept
: _M_value(__v), _M_cat(&__cat) { }
template<typename _ErrorConditionEnum,
typename = _Check<_ErrorConditionEnum>>
error_condition(_ErrorConditionEnum __e) noexcept
{
using __adl_only::make_error_condition;
*this = make_error_condition(__e);
}
error_condition(const error_condition&) = default;
error_condition& operator=(const error_condition&) = default;
void
assign(int __v, const error_category& __cat) noexcept
{
_M_value = __v;
_M_cat = &__cat;
}
void
clear() noexcept
{ assign(0, generic_category()); }
[[__nodiscard__]]
int
value() const noexcept { return _M_value; }
[[__nodiscard__]]
const error_category&
category() const noexcept { return *_M_cat; }
__attribute ((__abi_tag__ ("cxx11")))
string
message() const
{ return category().message(value()); }
[[__nodiscard__]]
explicit operator bool() const noexcept
{ return _M_value != 0; }
private:
int _M_value;
const error_category* _M_cat;
};
[[__nodiscard__]]
inline error_condition
make_error_condition(errc __e) noexcept
{ return error_condition(static_cast<int>(__e), generic_category()); }
[[__nodiscard__]]
inline bool
operator==(const error_code& __lhs, const error_code& __rhs) noexcept
{
return __lhs.category() == __rhs.category()
&& __lhs.value() == __rhs.value();
}
[[__nodiscard__]]
inline bool
operator==(const error_code& __lhs, const error_condition& __rhs) noexcept
{
return __lhs.category().equivalent(__lhs.value(), __rhs)
|| __rhs.category().equivalent(__lhs, __rhs.value());
}
[[__nodiscard__]]
inline bool
operator==(const error_condition& __lhs,
const error_condition& __rhs) noexcept
{
return __lhs.category() == __rhs.category()
&& __lhs.value() == __rhs.value();
}
inline bool
operator<(const error_condition& __lhs,
const error_condition& __rhs) noexcept
{
return (__lhs.category() < __rhs.category()
|| (__lhs.category() == __rhs.category()
&& __lhs.value() < __rhs.value()));
}
inline bool
operator==(const error_condition& __lhs, const error_code& __rhs) noexcept
{
return (__rhs.category().equivalent(__rhs.value(), __lhs)
|| __lhs.category().equivalent(__rhs, __lhs.value()));
}
inline bool
operator!=(const error_code& __lhs, const error_code& __rhs) noexcept
{ return !(__lhs == __rhs); }
inline bool
operator!=(const error_code& __lhs, const error_condition& __rhs) noexcept
{ return !(__lhs == __rhs); }
inline bool
operator!=(const error_condition& __lhs, const error_code& __rhs) noexcept
{ return !(__lhs == __rhs); }
inline bool
operator!=(const error_condition& __lhs,
const error_condition& __rhs) noexcept
{ return !(__lhs == __rhs); }
class system_error : public std::runtime_error
{
private:
error_code _M_code;
public:
system_error(error_code __ec = error_code())
: runtime_error(__ec.message()), _M_code(__ec) { }
system_error(error_code __ec, const string& __what)
: runtime_error(__what + ": " + __ec.message()), _M_code(__ec) { }
system_error(error_code __ec, const char* __what)
: runtime_error(__what + (": " + __ec.message())), _M_code(__ec) { }
system_error(int __v, const error_category& __ecat, const char* __what)
: system_error(error_code(__v, __ecat), __what) { }
system_error(int __v, const error_category& __ecat)
: runtime_error(error_code(__v, __ecat).message()),
_M_code(__v, __ecat) { }
system_error(int __v, const error_category& __ecat, const string& __what)
: runtime_error(__what + ": " + error_code(__v, __ecat).message()),
_M_code(__v, __ecat) { }
system_error (const system_error &) = default;
system_error &operator= (const system_error &) = default;
virtual ~system_error() noexcept;
const error_code&
code() const noexcept { return _M_code; }
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<>
struct hash<error_code>
: public __hash_base<size_t, error_code>
{
size_t
operator()(const error_code& __e) const noexcept
{
const size_t __tmp = std::_Hash_impl::hash(__e.value());
return std::_Hash_impl::__hash_combine(&__e.category(), __tmp);
}
};
template<>
struct hash<error_condition>
: public __hash_base<size_t, error_condition>
{
size_t
operator()(const error_condition& __e) const noexcept
{
const size_t __tmp = std::_Hash_impl::hash(__e.value());
return std::_Hash_impl::__hash_combine(&__e.category(), __tmp);
}
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
enum _Ios_Fmtflags
{
_S_boolalpha = 1L << 0,
_S_dec = 1L << 1,
_S_fixed = 1L << 2,
_S_hex = 1L << 3,
_S_internal = 1L << 4,
_S_left = 1L << 5,
_S_oct = 1L << 6,
_S_right = 1L << 7,
_S_scientific = 1L << 8,
_S_showbase = 1L << 9,
_S_showpoint = 1L << 10,
_S_showpos = 1L << 11,
_S_skipws = 1L << 12,
_S_unitbuf = 1L << 13,
_S_uppercase = 1L << 14,
_S_adjustfield = _S_left | _S_right | _S_internal,
_S_basefield = _S_dec | _S_oct | _S_hex,
_S_floatfield = _S_scientific | _S_fixed,
_S_ios_fmtflags_end = 1L << 16,
_S_ios_fmtflags_max = 0x7fffffff,
_S_ios_fmtflags_min = ~0x7fffffff
};
inline constexpr _Ios_Fmtflags
operator&(_Ios_Fmtflags __a, _Ios_Fmtflags __b)
{ return _Ios_Fmtflags(static_cast<int>(__a) & static_cast<int>(__b)); }
inline constexpr _Ios_Fmtflags
operator|(_Ios_Fmtflags __a, _Ios_Fmtflags __b)
{ return _Ios_Fmtflags(static_cast<int>(__a) | static_cast<int>(__b)); }
inline constexpr _Ios_Fmtflags
operator^(_Ios_Fmtflags __a, _Ios_Fmtflags __b)
{ return _Ios_Fmtflags(static_cast<int>(__a) ^ static_cast<int>(__b)); }
inline constexpr _Ios_Fmtflags
operator~(_Ios_Fmtflags __a)
{ return _Ios_Fmtflags(~static_cast<int>(__a)); }
inline const _Ios_Fmtflags&
operator|=(_Ios_Fmtflags& __a, _Ios_Fmtflags __b)
{ return __a = __a | __b; }
inline const _Ios_Fmtflags&
operator&=(_Ios_Fmtflags& __a, _Ios_Fmtflags __b)
{ return __a = __a & __b; }
inline const _Ios_Fmtflags&
operator^=(_Ios_Fmtflags& __a, _Ios_Fmtflags __b)
{ return __a = __a ^ __b; }
enum _Ios_Openmode
{
_S_app = 1L << 0,
_S_ate = 1L << 1,
_S_bin = 1L << 2,
_S_in = 1L << 3,
_S_out = 1L << 4,
_S_trunc = 1L << 5,
_S_noreplace = 1L << 6,
_S_ios_openmode_end = 1L << 16,
_S_ios_openmode_max = 0x7fffffff,
_S_ios_openmode_min = ~0x7fffffff
};
inline constexpr _Ios_Openmode
operator&(_Ios_Openmode __a, _Ios_Openmode __b)
{ return _Ios_Openmode(static_cast<int>(__a) & static_cast<int>(__b)); }
inline constexpr _Ios_Openmode
operator|(_Ios_Openmode __a, _Ios_Openmode __b)
{ return _Ios_Openmode(static_cast<int>(__a) | static_cast<int>(__b)); }
inline constexpr _Ios_Openmode
operator^(_Ios_Openmode __a, _Ios_Openmode __b)
{ return _Ios_Openmode(static_cast<int>(__a) ^ static_cast<int>(__b)); }
inline constexpr _Ios_Openmode
operator~(_Ios_Openmode __a)
{ return _Ios_Openmode(~static_cast<int>(__a)); }
inline const _Ios_Openmode&
operator|=(_Ios_Openmode& __a, _Ios_Openmode __b)
{ return __a = __a | __b; }
inline const _Ios_Openmode&
operator&=(_Ios_Openmode& __a, _Ios_Openmode __b)
{ return __a = __a & __b; }
inline const _Ios_Openmode&
operator^=(_Ios_Openmode& __a, _Ios_Openmode __b)
{ return __a = __a ^ __b; }
enum _Ios_Iostate
{
_S_goodbit = 0,
_S_badbit = 1L << 0,
_S_eofbit = 1L << 1,
_S_failbit = 1L << 2,
_S_ios_iostate_end = 1L << 16,
_S_ios_iostate_max = 0x7fffffff,
_S_ios_iostate_min = ~0x7fffffff
};
inline constexpr _Ios_Iostate
operator&(_Ios_Iostate __a, _Ios_Iostate __b)
{ return _Ios_Iostate(static_cast<int>(__a) & static_cast<int>(__b)); }
inline constexpr _Ios_Iostate
operator|(_Ios_Iostate __a, _Ios_Iostate __b)
{ return _Ios_Iostate(static_cast<int>(__a) | static_cast<int>(__b)); }
inline constexpr _Ios_Iostate
operator^(_Ios_Iostate __a, _Ios_Iostate __b)
{ return _Ios_Iostate(static_cast<int>(__a) ^ static_cast<int>(__b)); }
inline constexpr _Ios_Iostate
operator~(_Ios_Iostate __a)
{ return _Ios_Iostate(~static_cast<int>(__a)); }
inline const _Ios_Iostate&
operator|=(_Ios_Iostate& __a, _Ios_Iostate __b)
{ return __a = __a | __b; }
inline const _Ios_Iostate&
operator&=(_Ios_Iostate& __a, _Ios_Iostate __b)
{ return __a = __a & __b; }
inline const _Ios_Iostate&
operator^=(_Ios_Iostate& __a, _Ios_Iostate __b)
{ return __a = __a ^ __b; }
enum _Ios_Seekdir
{
_S_beg = 0,
_S_cur = 1,
_S_end = 2,
_S_ios_seekdir_end = 1L << 16
};
enum class io_errc { stream = 1 };
template <> struct is_error_code_enum<io_errc> : public true_type { };
[[__nodiscard__, __gnu__::__const__]]
const error_category&
iostream_category() noexcept;
[[__nodiscard__]]
inline error_code
make_error_code(io_errc __e) noexcept
{ return error_code(static_cast<int>(__e), iostream_category()); }
[[__nodiscard__]]
inline error_condition
make_error_condition(io_errc __e) noexcept
{ return error_condition(static_cast<int>(__e), iostream_category()); }
class ios_base
{
public:
class __attribute ((__abi_tag__ ("cxx11"))) failure : public system_error
{
public:
explicit
failure(const string& __str);
explicit
failure(const string&, const error_code&);
explicit
failure(const char*, const error_code& = io_errc::stream);
virtual
~failure() throw();
virtual const char*
what() const throw();
};
typedef _Ios_Fmtflags fmtflags;
static const fmtflags boolalpha = _S_boolalpha;
static const fmtflags dec = _S_dec;
static const fmtflags fixed = _S_fixed;
static const fmtflags hex = _S_hex;
static const fmtflags internal = _S_internal;
static const fmtflags left = _S_left;
static const fmtflags oct = _S_oct;
static const fmtflags right = _S_right;
static const fmtflags scientific = _S_scientific;
static const fmtflags showbase = _S_showbase;
static const fmtflags showpoint = _S_showpoint;
static const fmtflags showpos = _S_showpos;
static const fmtflags skipws = _S_skipws;
static const fmtflags unitbuf = _S_unitbuf;
static const fmtflags uppercase = _S_uppercase;
static const fmtflags adjustfield = _S_adjustfield;
static const fmtflags basefield = _S_basefield;
static const fmtflags floatfield = _S_floatfield;
typedef _Ios_Iostate iostate;
static const iostate badbit = _S_badbit;
static const iostate eofbit = _S_eofbit;
static const iostate failbit = _S_failbit;
static const iostate goodbit = _S_goodbit;
typedef _Ios_Openmode openmode;
static const openmode app = _S_app;
static const openmode ate = _S_ate;
static const openmode binary = _S_bin;
static const openmode in = _S_in;
static const openmode out = _S_out;
static const openmode trunc = _S_trunc;
static const openmode __noreplace = _S_noreplace;
typedef _Ios_Seekdir seekdir;
static const seekdir beg = _S_beg;
static const seekdir cur = _S_cur;
static const seekdir end = _S_end;
enum event
{
erase_event,
imbue_event,
copyfmt_event
};
typedef void (*event_callback) (event __e, ios_base& __b, int __i);
void
register_callback(event_callback __fn, int __index);
protected:
streamsize _M_precision;
streamsize _M_width;
fmtflags _M_flags;
iostate _M_exception;
iostate _M_streambuf_state;
struct _Callback_list
{
_Callback_list* _M_next;
ios_base::event_callback _M_fn;
int _M_index;
_Atomic_word _M_refcount;
_Callback_list(ios_base::event_callback __fn, int __index,
_Callback_list* __cb)
: _M_next(__cb), _M_fn(__fn), _M_index(__index), _M_refcount(0) { }
void
_M_add_reference() { __gnu_cxx::__atomic_add_dispatch(&_M_refcount, 1); }
int
_M_remove_reference()
{
;
int __res = __gnu_cxx::__exchange_and_add_dispatch(&_M_refcount, -1);
if (__res == 0)
{
;
}
return __res;
}
};
_Callback_list* _M_callbacks;
void
_M_call_callbacks(event __ev) throw();
void
_M_dispose_callbacks(void) throw();
struct _Words
{
void* _M_pword;
long _M_iword;
_Words() : _M_pword(0), _M_iword(0) { }
};
_Words _M_word_zero;
enum { _S_local_word_size = 8 };
_Words _M_local_word[_S_local_word_size];
int _M_word_size;
_Words* _M_word;
_Words&
_M_grow_words(int __index, bool __iword);
locale _M_ios_locale;
void
_M_init() throw();
public:
class Init
{
friend class ios_base;
public:
Init();
~Init();
Init(const Init&) = default;
Init& operator=(const Init&) = default;
private:
static _Atomic_word _S_refcount;
static bool _S_synced_with_stdio;
};
fmtflags
flags() const
{ return _M_flags; }
fmtflags
flags(fmtflags __fmtfl)
{
fmtflags __old = _M_flags;
_M_flags = __fmtfl;
return __old;
}
fmtflags
setf(fmtflags __fmtfl)
{
fmtflags __old = _M_flags;
_M_flags |= __fmtfl;
return __old;
}
fmtflags
setf(fmtflags __fmtfl, fmtflags __mask)
{
fmtflags __old = _M_flags;
_M_flags &= ~__mask;
_M_flags |= (__fmtfl & __mask);
return __old;
}
void
unsetf(fmtflags __mask)
{ _M_flags &= ~__mask; }
streamsize
precision() const
{ return _M_precision; }
streamsize
precision(streamsize __prec)
{
streamsize __old = _M_precision;
_M_precision = __prec;
return __old;
}
streamsize
width() const
{ return _M_width; }
streamsize
width(streamsize __wide)
{
streamsize __old = _M_width;
_M_width = __wide;
return __old;
}
static bool
sync_with_stdio(bool __sync = true);
locale
imbue(const locale& __loc) throw();
locale
getloc() const
{ return _M_ios_locale; }
const locale&
_M_getloc() const
{ return _M_ios_locale; }
static int
xalloc() throw();
long&
iword(int __ix)
{
_Words& __word = ((unsigned)__ix < (unsigned)_M_word_size)
? _M_word[__ix] : _M_grow_words(__ix, true);
return __word._M_iword;
}
void*&
pword(int __ix)
{
_Words& __word = ((unsigned)__ix < (unsigned)_M_word_size)
? _M_word[__ix] : _M_grow_words(__ix, false);
return __word._M_pword;
}
virtual ~ios_base();
protected:
ios_base() throw ();
public:
ios_base(const ios_base&) = delete;
ios_base&
operator=(const ios_base&) = delete;
protected:
void
_M_move(ios_base&) noexcept;
void
_M_swap(ios_base& __rhs) noexcept;
};
inline ios_base&
boolalpha(ios_base& __base)
{
__base.setf(ios_base::boolalpha);
return __base;
}
inline ios_base&
noboolalpha(ios_base& __base)
{
__base.unsetf(ios_base::boolalpha);
return __base;
}
inline ios_base&
showbase(ios_base& __base)
{
__base.setf(ios_base::showbase);
return __base;
}
inline ios_base&
noshowbase(ios_base& __base)
{
__base.unsetf(ios_base::showbase);
return __base;
}
inline ios_base&
showpoint(ios_base& __base)
{
__base.setf(ios_base::showpoint);
return __base;
}
inline ios_base&
noshowpoint(ios_base& __base)
{
__base.unsetf(ios_base::showpoint);
return __base;
}
inline ios_base&
showpos(ios_base& __base)
{
__base.setf(ios_base::showpos);
return __base;
}
inline ios_base&
noshowpos(ios_base& __base)
{
__base.unsetf(ios_base::showpos);
return __base;
}
inline ios_base&
skipws(ios_base& __base)
{
__base.setf(ios_base::skipws);
return __base;
}
inline ios_base&
noskipws(ios_base& __base)
{
__base.unsetf(ios_base::skipws);
return __base;
}
inline ios_base&
uppercase(ios_base& __base)
{
__base.setf(ios_base::uppercase);
return __base;
}
inline ios_base&
nouppercase(ios_base& __base)
{
__base.unsetf(ios_base::uppercase);
return __base;
}
inline ios_base&
unitbuf(ios_base& __base)
{
__base.setf(ios_base::unitbuf);
return __base;
}
inline ios_base&
nounitbuf(ios_base& __base)
{
__base.unsetf(ios_base::unitbuf);
return __base;
}
inline ios_base&
internal(ios_base& __base)
{
__base.setf(ios_base::internal, ios_base::adjustfield);
return __base;
}
inline ios_base&
left(ios_base& __base)
{
__base.setf(ios_base::left, ios_base::adjustfield);
return __base;
}
inline ios_base&
right(ios_base& __base)
{
__base.setf(ios_base::right, ios_base::adjustfield);
return __base;
}
inline ios_base&
dec(ios_base& __base)
{
__base.setf(ios_base::dec, ios_base::basefield);
return __base;
}
inline ios_base&
hex(ios_base& __base)
{
__base.setf(ios_base::hex, ios_base::basefield);
return __base;
}
inline ios_base&
oct(ios_base& __base)
{
__base.setf(ios_base::oct, ios_base::basefield);
return __base;
}
inline ios_base&
fixed(ios_base& __base)
{
__base.setf(ios_base::fixed, ios_base::floatfield);
return __base;
}
inline ios_base&
scientific(ios_base& __base)
{
__base.setf(ios_base::scientific, ios_base::floatfield);
return __base;
}
inline ios_base&
hexfloat(ios_base& __base)
{
__base.setf(ios_base::fixed | ios_base::scientific, ios_base::floatfield);
return __base;
}
inline ios_base&
defaultfloat(ios_base& __base)
{
__base.unsetf(ios_base::floatfield);
return __base;
}
}
typedef unsigned long int wctype_t;
enum
{
__ISwupper = 0,
__ISwlower = 1,
__ISwalpha = 2,
__ISwdigit = 3,
__ISwxdigit = 4,
__ISwspace = 5,
__ISwprint = 6,
__ISwgraph = 7,
__ISwblank = 8,
__ISwcntrl = 9,
__ISwpunct = 10,
__ISwalnum = 11,
_ISwupper = ((__ISwupper) < 8 ? (int) ((1UL << (__ISwupper)) << 24) : ((__ISwupper) < 16 ? (int) ((1UL << (__ISwupper)) << 8) : ((__ISwupper) < 24 ? (int) ((1UL << (__ISwupper)) >> 8) : (int) ((1UL << (__ISwupper)) >> 24)))),
_ISwlower = ((__ISwlower) < 8 ? (int) ((1UL << (__ISwlower)) << 24) : ((__ISwlower) < 16 ? (int) ((1UL << (__ISwlower)) << 8) : ((__ISwlower) < 24 ? (int) ((1UL << (__ISwlower)) >> 8) : (int) ((1UL << (__ISwlower)) >> 24)))),
_ISwalpha = ((__ISwalpha) < 8 ? (int) ((1UL << (__ISwalpha)) << 24) : ((__ISwalpha) < 16 ? (int) ((1UL << (__ISwalpha)) << 8) : ((__ISwalpha) < 24 ? (int) ((1UL << (__ISwalpha)) >> 8) : (int) ((1UL << (__ISwalpha)) >> 24)))),
_ISwdigit = ((__ISwdigit) < 8 ? (int) ((1UL << (__ISwdigit)) << 24) : ((__ISwdigit) < 16 ? (int) ((1UL << (__ISwdigit)) << 8) : ((__ISwdigit) < 24 ? (int) ((1UL << (__ISwdigit)) >> 8) : (int) ((1UL << (__ISwdigit)) >> 24)))),
_ISwxdigit = ((__ISwxdigit) < 8 ? (int) ((1UL << (__ISwxdigit)) << 24) : ((__ISwxdigit) < 16 ? (int) ((1UL << (__ISwxdigit)) << 8) : ((__ISwxdigit) < 24 ? (int) ((1UL << (__ISwxdigit)) >> 8) : (int) ((1UL << (__ISwxdigit)) >> 24)))),
_ISwspace = ((__ISwspace) < 8 ? (int) ((1UL << (__ISwspace)) << 24) : ((__ISwspace) < 16 ? (int) ((1UL << (__ISwspace)) << 8) : ((__ISwspace) < 24 ? (int) ((1UL << (__ISwspace)) >> 8) : (int) ((1UL << (__ISwspace)) >> 24)))),
_ISwprint = ((__ISwprint) < 8 ? (int) ((1UL << (__ISwprint)) << 24) : ((__ISwprint) < 16 ? (int) ((1UL << (__ISwprint)) << 8) : ((__ISwprint) < 24 ? (int) ((1UL << (__ISwprint)) >> 8) : (int) ((1UL << (__ISwprint)) >> 24)))),
_ISwgraph = ((__ISwgraph) < 8 ? (int) ((1UL << (__ISwgraph)) << 24) : ((__ISwgraph) < 16 ? (int) ((1UL << (__ISwgraph)) << 8) : ((__ISwgraph) < 24 ? (int) ((1UL << (__ISwgraph)) >> 8) : (int) ((1UL << (__ISwgraph)) >> 24)))),
_ISwblank = ((__ISwblank) < 8 ? (int) ((1UL << (__ISwblank)) << 24) : ((__ISwblank) < 16 ? (int) ((1UL << (__ISwblank)) << 8) : ((__ISwblank) < 24 ? (int) ((1UL << (__ISwblank)) >> 8) : (int) ((1UL << (__ISwblank)) >> 24)))),
_ISwcntrl = ((__ISwcntrl) < 8 ? (int) ((1UL << (__ISwcntrl)) << 24) : ((__ISwcntrl) < 16 ? (int) ((1UL << (__ISwcntrl)) << 8) : ((__ISwcntrl) < 24 ? (int) ((1UL << (__ISwcntrl)) >> 8) : (int) ((1UL << (__ISwcntrl)) >> 24)))),
_ISwpunct = ((__ISwpunct) < 8 ? (int) ((1UL << (__ISwpunct)) << 24) : ((__ISwpunct) < 16 ? (int) ((1UL << (__ISwpunct)) << 8) : ((__ISwpunct) < 24 ? (int) ((1UL << (__ISwpunct)) >> 8) : (int) ((1UL << (__ISwpunct)) >> 24)))),
_ISwalnum = ((__ISwalnum) < 8 ? (int) ((1UL << (__ISwalnum)) << 24) : ((__ISwalnum) < 16 ? (int) ((1UL << (__ISwalnum)) << 8) : ((__ISwalnum) < 24 ? (int) ((1UL << (__ISwalnum)) >> 8) : (int) ((1UL << (__ISwalnum)) >> 24))))
};
extern "C" {
extern int iswalnum (wint_t __wc) noexcept (true);
extern int iswalpha (wint_t __wc) noexcept (true);
extern int iswcntrl (wint_t __wc) noexcept (true);
extern int iswdigit (wint_t __wc) noexcept (true);
extern int iswgraph (wint_t __wc) noexcept (true);
extern int iswlower (wint_t __wc) noexcept (true);
extern int iswprint (wint_t __wc) noexcept (true);
extern int iswpunct (wint_t __wc) noexcept (true);
extern int iswspace (wint_t __wc) noexcept (true);
extern int iswupper (wint_t __wc) noexcept (true);
extern int iswxdigit (wint_t __wc) noexcept (true);
extern int iswblank (wint_t __wc) noexcept (true);
extern wctype_t wctype (const char *__property) noexcept (true);
extern int iswctype (wint_t __wc, wctype_t __desc) noexcept (true);
extern wint_t towlower (wint_t __wc) noexcept (true);
extern wint_t towupper (wint_t __wc) noexcept (true);
}
extern "C" {
typedef const __int32_t *wctrans_t;
extern wctrans_t wctrans (const char *__property) noexcept (true);
extern wint_t towctrans (wint_t __wc, wctrans_t __desc) noexcept (true);
extern int iswalnum_l (wint_t __wc, locale_t __locale) noexcept (true);
extern int iswalpha_l (wint_t __wc, locale_t __locale) noexcept (true);
extern int iswcntrl_l (wint_t __wc, locale_t __locale) noexcept (true);
extern int iswdigit_l (wint_t __wc, locale_t __locale) noexcept (true);
extern int iswgraph_l (wint_t __wc, locale_t __locale) noexcept (true);
extern int iswlower_l (wint_t __wc, locale_t __locale) noexcept (true);
extern int iswprint_l (wint_t __wc, locale_t __locale) noexcept (true);
extern int iswpunct_l (wint_t __wc, locale_t __locale) noexcept (true);
extern int iswspace_l (wint_t __wc, locale_t __locale) noexcept (true);
extern int iswupper_l (wint_t __wc, locale_t __locale) noexcept (true);
extern int iswxdigit_l (wint_t __wc, locale_t __locale) noexcept (true);
extern int iswblank_l (wint_t __wc, locale_t __locale) noexcept (true);
extern wctype_t wctype_l (const char *__property, locale_t __locale)
noexcept (true);
extern int iswctype_l (wint_t __wc, wctype_t __desc, locale_t __locale)
noexcept (true);
extern wint_t towlower_l (wint_t __wc, locale_t __locale) noexcept (true);
extern wint_t towupper_l (wint_t __wc, locale_t __locale) noexcept (true);
extern wctrans_t wctrans_l (const char *__property, locale_t __locale)
noexcept (true);
extern wint_t towctrans_l (wint_t __wc, wctrans_t __desc,
locale_t __locale) noexcept (true);
}
namespace std
{
using ::wctrans_t;
using ::wctype_t;
using ::wint_t;
using ::iswalnum;
using ::iswalpha;
using ::iswblank;
using ::iswcntrl;
using ::iswctype;
using ::iswdigit;
using ::iswgraph;
using ::iswlower;
using ::iswprint;
using ::iswpunct;
using ::iswspace;
using ::iswupper;
using ::iswxdigit;
using ::towctrans;
using ::towlower;
using ::towupper;
using ::wctrans;
using ::wctype;
}
namespace std __attribute__ ((__visibility__ ("default")))
{
struct ctype_base
{
typedef const int* __to_type;
typedef unsigned short mask;
static const mask upper = _ISupper;
static const mask lower = _ISlower;
static const mask alpha = _ISalpha;
static const mask digit = _ISdigit;
static const mask xdigit = _ISxdigit;
static const mask space = _ISspace;
static const mask print = _ISprint;
static const mask graph = _ISalpha | _ISdigit | _ISpunct;
static const mask cntrl = _IScntrl;
static const mask punct = _ISpunct;
static const mask alnum = _ISalpha | _ISdigit;
static const mask blank = _ISblank;
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _CharT, typename _Traits>
streamsize
__copy_streambufs_eof(basic_streambuf<_CharT, _Traits>*,
basic_streambuf<_CharT, _Traits>*, bool&);
template<typename _CharT, typename _Traits>
class basic_streambuf
{
public:
typedef _CharT char_type;
typedef _Traits traits_type;
typedef typename traits_type::int_type int_type;
typedef typename traits_type::pos_type pos_type;
typedef typename traits_type::off_type off_type;
typedef basic_streambuf<char_type, traits_type> __streambuf_type;
friend class basic_ios<char_type, traits_type>;
friend class basic_istream<char_type, traits_type>;
friend class basic_ostream<char_type, traits_type>;
friend class istreambuf_iterator<char_type, traits_type>;
friend class ostreambuf_iterator<char_type, traits_type>;
friend streamsize
__copy_streambufs_eof<>(basic_streambuf*, basic_streambuf*, bool&);
template<bool _IsMove, typename _CharT2>
friend typename __gnu_cxx::__enable_if<__is_char<_CharT2>::__value,
_CharT2*>::__type
__copy_move_a2(istreambuf_iterator<_CharT2>,
istreambuf_iterator<_CharT2>, _CharT2*);
template<typename _CharT2>
friend typename __gnu_cxx::__enable_if<__is_char<_CharT2>::__value,
istreambuf_iterator<_CharT2> >::__type
find(istreambuf_iterator<_CharT2>, istreambuf_iterator<_CharT2>,
const _CharT2&);
template<typename _CharT2, typename _Distance>
friend typename __gnu_cxx::__enable_if<__is_char<_CharT2>::__value,
void>::__type
advance(istreambuf_iterator<_CharT2>&, _Distance);
friend void __istream_extract(istream&, char*, streamsize);
template<typename _CharT2, typename _Traits2, typename _Alloc>
friend basic_istream<_CharT2, _Traits2>&
operator>>(basic_istream<_CharT2, _Traits2>&,
basic_string<_CharT2, _Traits2, _Alloc>&);
template<typename _CharT2, typename _Traits2, typename _Alloc>
friend basic_istream<_CharT2, _Traits2>&
getline(basic_istream<_CharT2, _Traits2>&,
basic_string<_CharT2, _Traits2, _Alloc>&, _CharT2);
protected:
char_type* _M_in_beg;
char_type* _M_in_cur;
char_type* _M_in_end;
char_type* _M_out_beg;
char_type* _M_out_cur;
char_type* _M_out_end;
locale _M_buf_locale;
public:
virtual
~basic_streambuf()
{ }
locale
pubimbue(const locale& __loc)
{
locale __tmp(this->getloc());
this->imbue(__loc);
_M_buf_locale = __loc;
return __tmp;
}
locale
getloc() const
{ return _M_buf_locale; }
basic_streambuf*
pubsetbuf(char_type* __s, streamsize __n)
{ return this->setbuf(__s, __n); }
pos_type
pubseekoff(off_type __off, ios_base::seekdir __way,
ios_base::openmode __mode = ios_base::in | ios_base::out)
{ return this->seekoff(__off, __way, __mode); }
pos_type
pubseekpos(pos_type __sp,
ios_base::openmode __mode = ios_base::in | ios_base::out)
{ return this->seekpos(__sp, __mode); }
int
pubsync() { return this->sync(); }
streamsize
in_avail()
{
const streamsize __ret = this->egptr() - this->gptr();
return __ret ? __ret : this->showmanyc();
}
int_type
snextc()
{
int_type __ret = traits_type::eof();
if (__builtin_expect(!traits_type::eq_int_type(this->sbumpc(),
__ret), true))
__ret = this->sgetc();
return __ret;
}
int_type
sbumpc()
{
int_type __ret;
if (__builtin_expect(this->gptr() < this->egptr(), true))
{
__ret = traits_type::to_int_type(*this->gptr());
this->gbump(1);
}
else
__ret = this->uflow();
return __ret;
}
int_type
sgetc()
{
int_type __ret;
if (__builtin_expect(this->gptr() < this->egptr(), true))
__ret = traits_type::to_int_type(*this->gptr());
else
__ret = this->underflow();
return __ret;
}
streamsize
sgetn(char_type* __s, streamsize __n)
{ return this->xsgetn(__s, __n); }
int_type
sputbackc(char_type __c)
{
int_type __ret;
const bool __testpos = this->eback() < this->gptr();
if (__builtin_expect(!__testpos ||
!traits_type::eq(__c, this->gptr()[-1]), false))
__ret = this->pbackfail(traits_type::to_int_type(__c));
else
{
this->gbump(-1);
__ret = traits_type::to_int_type(*this->gptr());
}
return __ret;
}
int_type
sungetc()
{
int_type __ret;
if (__builtin_expect(this->eback() < this->gptr(), true))
{
this->gbump(-1);
__ret = traits_type::to_int_type(*this->gptr());
}
else
__ret = this->pbackfail();
return __ret;
}
int_type
sputc(char_type __c)
{
int_type __ret;
if (__builtin_expect(this->pptr() < this->epptr(), true))
{
*this->pptr() = __c;
this->pbump(1);
__ret = traits_type::to_int_type(__c);
}
else
__ret = this->overflow(traits_type::to_int_type(__c));
return __ret;
}
streamsize
sputn(const char_type* __s, streamsize __n)
{ return this->xsputn(__s, __n); }
protected:
basic_streambuf()
: _M_in_beg(0), _M_in_cur(0), _M_in_end(0),
_M_out_beg(0), _M_out_cur(0), _M_out_end(0),
_M_buf_locale(locale())
{ }
char_type*
eback() const { return _M_in_beg; }
char_type*
gptr() const { return _M_in_cur; }
char_type*
egptr() const { return _M_in_end; }
void
gbump(int __n) { _M_in_cur += __n; }
void
setg(char_type* __gbeg, char_type* __gnext, char_type* __gend)
{
_M_in_beg = __gbeg;
_M_in_cur = __gnext;
_M_in_end = __gend;
}
char_type*
pbase() const { return _M_out_beg; }
char_type*
pptr() const { return _M_out_cur; }
char_type*
epptr() const { return _M_out_end; }
void
pbump(int __n) { _M_out_cur += __n; }
void
setp(char_type* __pbeg, char_type* __pend)
{
_M_out_beg = _M_out_cur = __pbeg;
_M_out_end = __pend;
}
virtual void
imbue(const locale& __loc __attribute__ ((__unused__)))
{ }
virtual basic_streambuf<char_type,_Traits>*
setbuf(char_type*, streamsize)
{ return this; }
virtual pos_type
seekoff(off_type, ios_base::seekdir,
ios_base::openmode = ios_base::in | ios_base::out)
{ return pos_type(off_type(-1)); }
virtual pos_type
seekpos(pos_type,
ios_base::openmode = ios_base::in | ios_base::out)
{ return pos_type(off_type(-1)); }
virtual int
sync() { return 0; }
virtual streamsize
showmanyc() { return 0; }
virtual streamsize
xsgetn(char_type* __s, streamsize __n);
virtual int_type
underflow()
{ return traits_type::eof(); }
virtual int_type
uflow()
{
int_type __ret = traits_type::eof();
const bool __testeof = traits_type::eq_int_type(this->underflow(),
__ret);
if (!__testeof)
{
__ret = traits_type::to_int_type(*this->gptr());
this->gbump(1);
}
return __ret;
}
virtual int_type
pbackfail(int_type __c __attribute__ ((__unused__)) = traits_type::eof())
{ return traits_type::eof(); }
virtual streamsize
xsputn(const char_type* __s, streamsize __n);
virtual int_type
overflow(int_type __c __attribute__ ((__unused__)) = traits_type::eof())
{ return traits_type::eof(); }
void
__safe_gbump(streamsize __n) { _M_in_cur += __n; }
void
__safe_pbump(streamsize __n) { _M_out_cur += __n; }
protected:
basic_streambuf(const basic_streambuf&);
basic_streambuf&
operator=(const basic_streambuf&);
void
swap(basic_streambuf& __sb)
{
std::swap(_M_in_beg, __sb._M_in_beg);
std::swap(_M_in_cur, __sb._M_in_cur);
std::swap(_M_in_end, __sb._M_in_end);
std::swap(_M_out_beg, __sb._M_out_beg);
std::swap(_M_out_cur, __sb._M_out_cur);
std::swap(_M_out_end, __sb._M_out_end);
std::swap(_M_buf_locale, __sb._M_buf_locale);
}
};
template<typename _CharT, typename _Traits>
std::basic_streambuf<_CharT, _Traits>::
basic_streambuf(const basic_streambuf&) = default;
template<typename _CharT, typename _Traits>
std::basic_streambuf<_CharT, _Traits>&
std::basic_streambuf<_CharT, _Traits>::
operator=(const basic_streambuf&) = default;
template<>
streamsize
__copy_streambufs_eof(basic_streambuf<char>* __sbin,
basic_streambuf<char>* __sbout, bool& __ineof);
template<>
streamsize
__copy_streambufs_eof(basic_streambuf<wchar_t>* __sbin,
basic_streambuf<wchar_t>* __sbout, bool& __ineof);
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _CharT, typename _Traits>
streamsize
basic_streambuf<_CharT, _Traits>::
xsgetn(char_type* __s, streamsize __n)
{
streamsize __ret = 0;
while (__ret < __n)
{
const streamsize __buf_len = this->egptr() - this->gptr();
if (__buf_len)
{
const streamsize __remaining = __n - __ret;
const streamsize __len = std::min(__buf_len, __remaining);
traits_type::copy(__s, this->gptr(), __len);
__ret += __len;
__s += __len;
this->__safe_gbump(__len);
}
if (__ret < __n)
{
const int_type __c = this->uflow();
if (!traits_type::eq_int_type(__c, traits_type::eof()))
{
traits_type::assign(*__s++, traits_type::to_char_type(__c));
++__ret;
}
else
break;
}
}
return __ret;
}
template<typename _CharT, typename _Traits>
streamsize
basic_streambuf<_CharT, _Traits>::
xsputn(const char_type* __s, streamsize __n)
{
streamsize __ret = 0;
while (__ret < __n)
{
const streamsize __buf_len = this->epptr() - this->pptr();
if (__buf_len)
{
const streamsize __remaining = __n - __ret;
const streamsize __len = std::min(__buf_len, __remaining);
traits_type::copy(this->pptr(), __s, __len);
__ret += __len;
__s += __len;
this->__safe_pbump(__len);
}
if (__ret < __n)
{
int_type __c = this->overflow(traits_type::to_int_type(*__s));
if (!traits_type::eq_int_type(__c, traits_type::eof()))
{
++__ret;
++__s;
}
else
break;
}
}
return __ret;
}
template<typename _CharT, typename _Traits>
streamsize
__copy_streambufs_eof(basic_streambuf<_CharT, _Traits>* __sbin,
basic_streambuf<_CharT, _Traits>* __sbout,
bool& __ineof)
{
streamsize __ret = 0;
__ineof = true;
typename _Traits::int_type __c = __sbin->sgetc();
while (!_Traits::eq_int_type(__c, _Traits::eof()))
{
__c = __sbout->sputc(_Traits::to_char_type(__c));
if (_Traits::eq_int_type(__c, _Traits::eof()))
{
__ineof = false;
break;
}
++__ret;
__c = __sbin->snextc();
}
return __ret;
}
template<typename _CharT, typename _Traits>
inline streamsize
__copy_streambufs(basic_streambuf<_CharT, _Traits>* __sbin,
basic_streambuf<_CharT, _Traits>* __sbout)
{
bool __ineof;
return __copy_streambufs_eof(__sbin, __sbout, __ineof);
}
extern template class basic_streambuf<char>;
extern template
streamsize
__copy_streambufs(basic_streambuf<char>*,
basic_streambuf<char>*);
extern template class basic_streambuf<wchar_t>;
extern template
streamsize
__copy_streambufs(basic_streambuf<wchar_t>*,
basic_streambuf<wchar_t>*);
}
namespace std __attribute__ ((__visibility__ ("default")))
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _CharT, typename _Traits>
class istreambuf_iterator
: public iterator<input_iterator_tag, _CharT, typename _Traits::off_type,
_CharT*, _CharT>
{
public:
typedef _CharT char_type;
typedef _Traits traits_type;
typedef typename _Traits::int_type int_type;
typedef basic_streambuf<_CharT, _Traits> streambuf_type;
typedef basic_istream<_CharT, _Traits> istream_type;
template<typename _CharT2>
friend typename __gnu_cxx::__enable_if<__is_char<_CharT2>::__value,
ostreambuf_iterator<_CharT2> >::__type
copy(istreambuf_iterator<_CharT2>, istreambuf_iterator<_CharT2>,
ostreambuf_iterator<_CharT2>);
template<bool _IsMove, typename _CharT2>
friend typename __gnu_cxx::__enable_if<__is_char<_CharT2>::__value,
_CharT2*>::__type
__copy_move_a2(istreambuf_iterator<_CharT2>,
istreambuf_iterator<_CharT2>, _CharT2*);
template<typename _CharT2, typename _Size>
friend typename __gnu_cxx::__enable_if<__is_char<_CharT2>::__value,
_CharT2*>::__type
__copy_n_a(istreambuf_iterator<_CharT2>, _Size, _CharT2*, bool);
template<typename _CharT2>
friend typename __gnu_cxx::__enable_if<__is_char<_CharT2>::__value,
istreambuf_iterator<_CharT2> >::__type
find(istreambuf_iterator<_CharT2>, istreambuf_iterator<_CharT2>,
const _CharT2&);
template<typename _CharT2, typename _Distance>
friend typename __gnu_cxx::__enable_if<__is_char<_CharT2>::__value,
void>::__type
advance(istreambuf_iterator<_CharT2>&, _Distance);
private:
mutable streambuf_type* _M_sbuf;
int_type _M_c;
public:
constexpr istreambuf_iterator() noexcept
: _M_sbuf(0), _M_c(traits_type::eof()) { }
istreambuf_iterator(const istreambuf_iterator&) noexcept = default;
~istreambuf_iterator() = default;
istreambuf_iterator(istream_type& __s) noexcept
: _M_sbuf(__s.rdbuf()), _M_c(traits_type::eof()) { }
istreambuf_iterator(streambuf_type* __s) noexcept
: _M_sbuf(__s), _M_c(traits_type::eof()) { }
istreambuf_iterator&
operator=(const istreambuf_iterator&) noexcept = default;
[[__nodiscard__]]
char_type
operator*() const
{
int_type __c = _M_get();
return traits_type::to_char_type(__c);
}
istreambuf_iterator&
operator++()
{
;
_M_sbuf->sbumpc();
_M_c = traits_type::eof();
return *this;
}
istreambuf_iterator
operator++(int)
{
;
istreambuf_iterator __old = *this;
__old._M_c = _M_sbuf->sbumpc();
_M_c = traits_type::eof();
return __old;
}
[[__nodiscard__]]
bool
equal(const istreambuf_iterator& __b) const
{ return _M_at_eof() == __b._M_at_eof(); }
private:
int_type
_M_get() const
{
int_type __ret = _M_c;
if (_M_sbuf && _S_is_eof(__ret) && _S_is_eof(__ret = _M_sbuf->sgetc()))
_M_sbuf = 0;
return __ret;
}
bool
_M_at_eof() const
{ return _S_is_eof(_M_get()); }
static bool
_S_is_eof(int_type __c)
{
const int_type __eof = traits_type::eof();
return traits_type::eq_int_type(__c, __eof);
}
};
template<typename _CharT, typename _Traits>
[[__nodiscard__]]
inline bool
operator==(const istreambuf_iterator<_CharT, _Traits>& __a,
const istreambuf_iterator<_CharT, _Traits>& __b)
{ return __a.equal(__b); }
template<typename _CharT, typename _Traits>
[[__nodiscard__]]
inline bool
operator!=(const istreambuf_iterator<_CharT, _Traits>& __a,
const istreambuf_iterator<_CharT, _Traits>& __b)
{ return !__a.equal(__b); }
template<typename _CharT, typename _Traits>
class ostreambuf_iterator
: public iterator<output_iterator_tag, void, void, void, void>
{
public:
typedef _CharT char_type;
typedef _Traits traits_type;
typedef basic_streambuf<_CharT, _Traits> streambuf_type;
typedef basic_ostream<_CharT, _Traits> ostream_type;
template<typename _CharT2>
friend typename __gnu_cxx::__enable_if<__is_char<_CharT2>::__value,
ostreambuf_iterator<_CharT2> >::__type
copy(istreambuf_iterator<_CharT2>, istreambuf_iterator<_CharT2>,
ostreambuf_iterator<_CharT2>);
private:
streambuf_type* _M_sbuf;
bool _M_failed;
public:
ostreambuf_iterator(ostream_type& __s) noexcept
: _M_sbuf(__s.rdbuf()), _M_failed(!_M_sbuf) { }
ostreambuf_iterator(streambuf_type* __s) noexcept
: _M_sbuf(__s), _M_failed(!_M_sbuf) { }
ostreambuf_iterator&
operator=(_CharT __c)
{
if (!_M_failed &&
_Traits::eq_int_type(_M_sbuf->sputc(__c), _Traits::eof()))
_M_failed = true;
return *this;
}
[[__nodiscard__]]
ostreambuf_iterator&
operator*()
{ return *this; }
ostreambuf_iterator&
operator++(int)
{ return *this; }
ostreambuf_iterator&
operator++()
{ return *this; }
[[__nodiscard__]]
bool
failed() const noexcept
{ return _M_failed; }
ostreambuf_iterator&
_M_put(const _CharT* __ws, streamsize __len)
{
if (__builtin_expect(!_M_failed, true)
&& __builtin_expect(this->_M_sbuf->sputn(__ws, __len) != __len,
false))
_M_failed = true;
return *this;
}
};
#pragma GCC diagnostic pop
template<typename _CharT>
typename __gnu_cxx::__enable_if<__is_char<_CharT>::__value,
ostreambuf_iterator<_CharT> >::__type
copy(istreambuf_iterator<_CharT> __first,
istreambuf_iterator<_CharT> __last,
ostreambuf_iterator<_CharT> __result)
{
if (__first._M_sbuf && !__last._M_sbuf && !__result._M_failed)
{
bool __ineof;
__copy_streambufs_eof(__first._M_sbuf, __result._M_sbuf, __ineof);
if (!__ineof)
__result._M_failed = true;
}
return __result;
}
template<bool _IsMove, typename _CharT>
typename __gnu_cxx::__enable_if<__is_char<_CharT>::__value,
ostreambuf_iterator<_CharT> >::__type
__copy_move_a2(_CharT* __first, _CharT* __last,
ostreambuf_iterator<_CharT> __result)
{
const streamsize __num = __last - __first;
if (__num > 0)
__result._M_put(__first, __num);
return __result;
}
template<bool _IsMove, typename _CharT>
typename __gnu_cxx::__enable_if<__is_char<_CharT>::__value,
ostreambuf_iterator<_CharT> >::__type
__copy_move_a2(const _CharT* __first, const _CharT* __last,
ostreambuf_iterator<_CharT> __result)
{
const streamsize __num = __last - __first;
if (__num > 0)
__result._M_put(__first, __num);
return __result;
}
template<bool _IsMove, typename _CharT>
typename __gnu_cxx::__enable_if<__is_char<_CharT>::__value,
_CharT*>::__type
__copy_move_a2(istreambuf_iterator<_CharT> __first,
istreambuf_iterator<_CharT> __last, _CharT* __result)
{
typedef istreambuf_iterator<_CharT> __is_iterator_type;
typedef typename __is_iterator_type::traits_type traits_type;
typedef typename __is_iterator_type::streambuf_type streambuf_type;
typedef typename traits_type::int_type int_type;
if (__first._M_sbuf && !__last._M_sbuf)
{
streambuf_type* __sb = __first._M_sbuf;
int_type __c = __sb->sgetc();
while (!traits_type::eq_int_type(__c, traits_type::eof()))
{
const streamsize __n = __sb->egptr() - __sb->gptr();
if (__n > 1)
{
traits_type::copy(__result, __sb->gptr(), __n);
__sb->__safe_gbump(__n);
__result += __n;
__c = __sb->underflow();
}
else
{
*__result++ = traits_type::to_char_type(__c);
__c = __sb->snextc();
}
}
}
return __result;
}
template<typename _CharT, typename _Size>
typename __gnu_cxx::__enable_if<__is_char<_CharT>::__value,
_CharT*>::__type
__copy_n_a(istreambuf_iterator<_CharT> __it, _Size __n, _CharT* __result,
bool __strict __attribute__((__unused__)))
{
if (__n == 0)
return __result;
;
_CharT* __beg = __result;
__result += __it._M_sbuf->sgetn(__beg, __n);
;
return __result;
}
template<typename _CharT>
typename __gnu_cxx::__enable_if<__is_char<_CharT>::__value,
istreambuf_iterator<_CharT> >::__type
find(istreambuf_iterator<_CharT> __first,
istreambuf_iterator<_CharT> __last, const _CharT& __val)
{
typedef istreambuf_iterator<_CharT> __is_iterator_type;
typedef typename __is_iterator_type::traits_type traits_type;
typedef typename __is_iterator_type::streambuf_type streambuf_type;
typedef typename traits_type::int_type int_type;
const int_type __eof = traits_type::eof();
if (__first._M_sbuf && !__last._M_sbuf)
{
const int_type __ival = traits_type::to_int_type(__val);
streambuf_type* __sb = __first._M_sbuf;
int_type __c = __sb->sgetc();
while (!traits_type::eq_int_type(__c, __eof)
&& !traits_type::eq_int_type(__c, __ival))
{
streamsize __n = __sb->egptr() - __sb->gptr();
if (__n > 1)
{
const _CharT* __p = traits_type::find(__sb->gptr(),
__n, __val);
if (__p)
__n = __p - __sb->gptr();
__sb->__safe_gbump(__n);
__c = __sb->sgetc();
}
else
__c = __sb->snextc();
}
__first._M_c = __eof;
}
return __first;
}
template<typename _CharT, typename _Distance>
typename __gnu_cxx::__enable_if<__is_char<_CharT>::__value,
void>::__type
advance(istreambuf_iterator<_CharT>& __i, _Distance __n)
{
if (__n == 0)
return;
do { if (std::__is_constant_evaluated() && !bool(__n > 0)) __builtin_unreachable(); } while (false);
;
typedef istreambuf_iterator<_CharT> __is_iterator_type;
typedef typename __is_iterator_type::traits_type traits_type;
typedef typename __is_iterator_type::streambuf_type streambuf_type;
typedef typename traits_type::int_type int_type;
const int_type __eof = traits_type::eof();
streambuf_type* __sb = __i._M_sbuf;
while (__n > 0)
{
streamsize __size = __sb->egptr() - __sb->gptr();
if (__size > __n)
{
__sb->__safe_gbump(__n);
break;
}
__sb->__safe_gbump(__size);
__n -= __size;
if (traits_type::eq_int_type(__sb->underflow(), __eof))
{
;
break;
}
}
__i._M_c = __eof;
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Tp>
void
__convert_to_v(const char*, _Tp&, ios_base::iostate&,
const __c_locale&) throw();
template<>
void
__convert_to_v(const char*, float&, ios_base::iostate&,
const __c_locale&) throw();
template<>
void
__convert_to_v(const char*, double&, ios_base::iostate&,
const __c_locale&) throw();
template<>
void
__convert_to_v(const char*, long double&, ios_base::iostate&,
const __c_locale&) throw();
template<typename _CharT, typename _Traits>
struct __pad
{
static void
_S_pad(ios_base& __io, _CharT __fill, _CharT* __news,
const _CharT* __olds, streamsize __newlen, streamsize __oldlen);
};
template<typename _CharT>
_CharT*
__add_grouping(_CharT* __s, _CharT __sep,
const char* __gbeg, size_t __gsize,
const _CharT* __first, const _CharT* __last);
template<typename _CharT>
inline
ostreambuf_iterator<_CharT>
__write(ostreambuf_iterator<_CharT> __s, const _CharT* __ws, int __len)
{
__s._M_put(__ws, __len);
return __s;
}
template<typename _CharT, typename _OutIter>
inline
_OutIter
__write(_OutIter __s, const _CharT* __ws, int __len)
{
for (int __j = 0; __j < __len; __j++, ++__s)
*__s = __ws[__j];
return __s;
}
template<typename _CharT>
class __ctype_abstract_base : public locale::facet, public ctype_base
{
public:
typedef _CharT char_type;
bool
is(mask __m, char_type __c) const
{ return this->do_is(__m, __c); }
const char_type*
is(const char_type *__lo, const char_type *__hi, mask *__vec) const
{ return this->do_is(__lo, __hi, __vec); }
const char_type*
scan_is(mask __m, const char_type* __lo, const char_type* __hi) const
{ return this->do_scan_is(__m, __lo, __hi); }
const char_type*
scan_not(mask __m, const char_type* __lo, const char_type* __hi) const
{ return this->do_scan_not(__m, __lo, __hi); }
char_type
toupper(char_type __c) const
{ return this->do_toupper(__c); }
const char_type*
toupper(char_type *__lo, const char_type* __hi) const
{ return this->do_toupper(__lo, __hi); }
char_type
tolower(char_type __c) const
{ return this->do_tolower(__c); }
const char_type*
tolower(char_type* __lo, const char_type* __hi) const
{ return this->do_tolower(__lo, __hi); }
char_type
widen(char __c) const
{ return this->do_widen(__c); }
const char*
widen(const char* __lo, const char* __hi, char_type* __to) const
{ return this->do_widen(__lo, __hi, __to); }
char
narrow(char_type __c, char __dfault) const
{ return this->do_narrow(__c, __dfault); }
const char_type*
narrow(const char_type* __lo, const char_type* __hi,
char __dfault, char* __to) const
{ return this->do_narrow(__lo, __hi, __dfault, __to); }
protected:
explicit
__ctype_abstract_base(size_t __refs = 0): facet(__refs) { }
virtual
~__ctype_abstract_base() { }
virtual bool
do_is(mask __m, char_type __c) const = 0;
virtual const char_type*
do_is(const char_type* __lo, const char_type* __hi,
mask* __vec) const = 0;
virtual const char_type*
do_scan_is(mask __m, const char_type* __lo,
const char_type* __hi) const = 0;
virtual const char_type*
do_scan_not(mask __m, const char_type* __lo,
const char_type* __hi) const = 0;
virtual char_type
do_toupper(char_type __c) const = 0;
virtual const char_type*
do_toupper(char_type* __lo, const char_type* __hi) const = 0;
virtual char_type
do_tolower(char_type __c) const = 0;
virtual const char_type*
do_tolower(char_type* __lo, const char_type* __hi) const = 0;
virtual char_type
do_widen(char __c) const = 0;
virtual const char*
do_widen(const char* __lo, const char* __hi, char_type* __to) const = 0;
virtual char
do_narrow(char_type __c, char __dfault) const = 0;
virtual const char_type*
do_narrow(const char_type* __lo, const char_type* __hi,
char __dfault, char* __to) const = 0;
};
template<typename _CharT>
class ctype : public __ctype_abstract_base<_CharT>
{
public:
typedef _CharT char_type;
typedef typename __ctype_abstract_base<_CharT>::mask mask;
static locale::id id;
explicit
ctype(size_t __refs = 0) : __ctype_abstract_base<_CharT>(__refs) { }
protected:
virtual
~ctype();
virtual bool
do_is(mask __m, char_type __c) const;
virtual const char_type*
do_is(const char_type* __lo, const char_type* __hi, mask* __vec) const;
virtual const char_type*
do_scan_is(mask __m, const char_type* __lo, const char_type* __hi) const;
virtual const char_type*
do_scan_not(mask __m, const char_type* __lo,
const char_type* __hi) const;
virtual char_type
do_toupper(char_type __c) const;
virtual const char_type*
do_toupper(char_type* __lo, const char_type* __hi) const;
virtual char_type
do_tolower(char_type __c) const;
virtual const char_type*
do_tolower(char_type* __lo, const char_type* __hi) const;
virtual char_type
do_widen(char __c) const;
virtual const char*
do_widen(const char* __lo, const char* __hi, char_type* __dest) const;
virtual char
do_narrow(char_type, char __dfault) const;
virtual const char_type*
do_narrow(const char_type* __lo, const char_type* __hi,
char __dfault, char* __to) const;
};
template<typename _CharT>
locale::id ctype<_CharT>::id;
template<typename _CharT, typename _Traits, typename _Alloc>
class ctype<basic_string<_CharT, _Traits, _Alloc> >;
template<>
class ctype<char> : public locale::facet, public ctype_base
{
public:
typedef char char_type;
protected:
__c_locale _M_c_locale_ctype;
bool _M_del;
__to_type _M_toupper;
__to_type _M_tolower;
const mask* _M_table;
mutable char _M_widen_ok;
mutable char _M_widen[1 + static_cast<unsigned char>(-1)];
mutable char _M_narrow[1 + static_cast<unsigned char>(-1)];
mutable char _M_narrow_ok;
public:
static locale::id id;
static const size_t table_size = 1 + static_cast<unsigned char>(-1);
explicit
ctype(const mask* __table = 0, bool __del = false, size_t __refs = 0);
explicit
ctype(__c_locale __cloc, const mask* __table = 0, bool __del = false,
size_t __refs = 0);
inline bool
is(mask __m, char __c) const;
inline const char*
is(const char* __lo, const char* __hi, mask* __vec) const;
inline const char*
scan_is(mask __m, const char* __lo, const char* __hi) const;
inline const char*
scan_not(mask __m, const char* __lo, const char* __hi) const;
char_type
toupper(char_type __c) const
{ return this->do_toupper(__c); }
const char_type*
toupper(char_type *__lo, const char_type* __hi) const
{ return this->do_toupper(__lo, __hi); }
char_type
tolower(char_type __c) const
{ return this->do_tolower(__c); }
const char_type*
tolower(char_type* __lo, const char_type* __hi) const
{ return this->do_tolower(__lo, __hi); }
char_type
widen(char __c) const
{
if (_M_widen_ok)
return _M_widen[static_cast<unsigned char>(__c)];
this->_M_widen_init();
return this->do_widen(__c);
}
const char*
widen(const char* __lo, const char* __hi, char_type* __to) const
{
if (_M_widen_ok == 1)
{
if (__builtin_expect(__hi != __lo, true))
__builtin_memcpy(__to, __lo, __hi - __lo);
return __hi;
}
if (!_M_widen_ok)
_M_widen_init();
return this->do_widen(__lo, __hi, __to);
}
char
narrow(char_type __c, char __dfault) const
{
if (_M_narrow[static_cast<unsigned char>(__c)])
return _M_narrow[static_cast<unsigned char>(__c)];
const char __t = do_narrow(__c, __dfault);
if (__t != __dfault)
_M_narrow[static_cast<unsigned char>(__c)] = __t;
return __t;
}
const char_type*
narrow(const char_type* __lo, const char_type* __hi,
char __dfault, char* __to) const
{
if (__builtin_expect(_M_narrow_ok == 1, true))
{
if (__builtin_expect(__hi != __lo, true))
__builtin_memcpy(__to, __lo, __hi - __lo);
return __hi;
}
if (!_M_narrow_ok)
_M_narrow_init();
return this->do_narrow(__lo, __hi, __dfault, __to);
}
const mask*
table() const throw()
{ return _M_table; }
static const mask*
classic_table() throw();
protected:
virtual
~ctype();
virtual char_type
do_toupper(char_type __c) const;
virtual const char_type*
do_toupper(char_type* __lo, const char_type* __hi) const;
virtual char_type
do_tolower(char_type __c) const;
virtual const char_type*
do_tolower(char_type* __lo, const char_type* __hi) const;
virtual char_type
do_widen(char __c) const
{ return __c; }
virtual const char*
do_widen(const char* __lo, const char* __hi, char_type* __to) const
{
if (__builtin_expect(__hi != __lo, true))
__builtin_memcpy(__to, __lo, __hi - __lo);
return __hi;
}
virtual char
do_narrow(char_type __c, char __dfault __attribute__((__unused__))) const
{ return __c; }
virtual const char_type*
do_narrow(const char_type* __lo, const char_type* __hi,
char __dfault __attribute__((__unused__)), char* __to) const
{
if (__builtin_expect(__hi != __lo, true))
__builtin_memcpy(__to, __lo, __hi - __lo);
return __hi;
}
private:
void _M_narrow_init() const;
void _M_widen_init() const;
};
template<>
class ctype<wchar_t> : public __ctype_abstract_base<wchar_t>
{
public:
typedef wchar_t char_type;
typedef wctype_t __wmask_type;
protected:
__c_locale _M_c_locale_ctype;
bool _M_narrow_ok;
char _M_narrow[128];
wint_t _M_widen[1 + static_cast<unsigned char>(-1)];
mask _M_bit[16];
__wmask_type _M_wmask[16];
public:
static locale::id id;
explicit
ctype(size_t __refs = 0);
explicit
ctype(__c_locale __cloc, size_t __refs = 0);
protected:
__wmask_type
_M_convert_to_wmask(const mask __m) const throw();
virtual
~ctype();
virtual bool
do_is(mask __m, char_type __c) const;
virtual const char_type*
do_is(const char_type* __lo, const char_type* __hi, mask* __vec) const;
virtual const char_type*
do_scan_is(mask __m, const char_type* __lo, const char_type* __hi) const;
virtual const char_type*
do_scan_not(mask __m, const char_type* __lo,
const char_type* __hi) const;
virtual char_type
do_toupper(char_type __c) const;
virtual const char_type*
do_toupper(char_type* __lo, const char_type* __hi) const;
virtual char_type
do_tolower(char_type __c) const;
virtual const char_type*
do_tolower(char_type* __lo, const char_type* __hi) const;
virtual char_type
do_widen(char __c) const;
virtual const char*
do_widen(const char* __lo, const char* __hi, char_type* __to) const;
virtual char
do_narrow(char_type __c, char __dfault) const;
virtual const char_type*
do_narrow(const char_type* __lo, const char_type* __hi,
char __dfault, char* __to) const;
void
_M_initialize_ctype() throw();
};
template<typename _CharT>
class ctype_byname : public ctype<_CharT>
{
public:
typedef typename ctype<_CharT>::mask mask;
explicit
ctype_byname(const char* __s, size_t __refs = 0);
explicit
ctype_byname(const string& __s, size_t __refs = 0)
: ctype_byname(__s.c_str(), __refs) { }
protected:
virtual
~ctype_byname() { }
};
template<>
class ctype_byname<char> : public ctype<char>
{
public:
explicit
ctype_byname(const char* __s, size_t __refs = 0);
explicit
ctype_byname(const string& __s, size_t __refs = 0);
protected:
virtual
~ctype_byname();
};
template<>
class ctype_byname<wchar_t> : public ctype<wchar_t>
{
public:
explicit
ctype_byname(const char* __s, size_t __refs = 0);
explicit
ctype_byname(const string& __s, size_t __refs = 0);
protected:
virtual
~ctype_byname();
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
bool
ctype<char>::
is(mask __m, char __c) const
{ return _M_table[static_cast<unsigned char>(__c)] & __m; }
const char*
ctype<char>::
is(const char* __low, const char* __high, mask* __vec) const
{
while (__low < __high)
*__vec++ = _M_table[static_cast<unsigned char>(*__low++)];
return __high;
}
const char*
ctype<char>::
scan_is(mask __m, const char* __low, const char* __high) const
{
while (__low < __high
&& !(_M_table[static_cast<unsigned char>(*__low)] & __m))
++__low;
return __low;
}
const char*
ctype<char>::
scan_not(mask __m, const char* __low, const char* __high) const
{
while (__low < __high
&& (_M_table[static_cast<unsigned char>(*__low)] & __m) != 0)
++__low;
return __low;
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
class __num_base
{
public:
enum
{
_S_ominus,
_S_oplus,
_S_ox,
_S_oX,
_S_odigits,
_S_odigits_end = _S_odigits + 16,
_S_oudigits = _S_odigits_end,
_S_oudigits_end = _S_oudigits + 16,
_S_oe = _S_odigits + 14,
_S_oE = _S_oudigits + 14,
_S_oend = _S_oudigits_end
};
static const char* _S_atoms_out;
static const char* _S_atoms_in;
enum
{
_S_iminus,
_S_iplus,
_S_ix,
_S_iX,
_S_izero,
_S_ie = _S_izero + 14,
_S_iE = _S_izero + 20,
_S_iend = 26
};
static void
_S_format_float(const ios_base& __io, char* __fptr, char __mod) throw();
};
template<typename _CharT>
struct __numpunct_cache : public locale::facet
{
const char* _M_grouping;
size_t _M_grouping_size;
bool _M_use_grouping;
const _CharT* _M_truename;
size_t _M_truename_size;
const _CharT* _M_falsename;
size_t _M_falsename_size;
_CharT _M_decimal_point;
_CharT _M_thousands_sep;
_CharT _M_atoms_out[__num_base::_S_oend];
_CharT _M_atoms_in[__num_base::_S_iend];
bool _M_allocated;
__numpunct_cache(size_t __refs = 0)
: facet(__refs), _M_grouping(0), _M_grouping_size(0),
_M_use_grouping(false),
_M_truename(0), _M_truename_size(0), _M_falsename(0),
_M_falsename_size(0), _M_decimal_point(_CharT()),
_M_thousands_sep(_CharT()), _M_allocated(false)
{ }
~__numpunct_cache();
void
_M_cache(const locale& __loc);
private:
__numpunct_cache&
operator=(const __numpunct_cache&);
explicit
__numpunct_cache(const __numpunct_cache&);
};
template<typename _CharT>
__numpunct_cache<_CharT>::~__numpunct_cache()
{
if (_M_allocated)
{
delete [] _M_grouping;
delete [] _M_truename;
delete [] _M_falsename;
}
}
namespace __cxx11 {
template<typename _CharT>
class numpunct : public locale::facet
{
public:
typedef _CharT char_type;
typedef basic_string<_CharT> string_type;
typedef __numpunct_cache<_CharT> __cache_type;
protected:
__cache_type* _M_data;
public:
static locale::id id;
explicit
numpunct(size_t __refs = 0)
: facet(__refs), _M_data(0)
{ _M_initialize_numpunct(); }
explicit
numpunct(__cache_type* __cache, size_t __refs = 0)
: facet(__refs), _M_data(__cache)
{ _M_initialize_numpunct(); }
explicit
numpunct(__c_locale __cloc, size_t __refs = 0)
: facet(__refs), _M_data(0)
{ _M_initialize_numpunct(__cloc); }
char_type
decimal_point() const
{ return this->do_decimal_point(); }
char_type
thousands_sep() const
{ return this->do_thousands_sep(); }
string
grouping() const
{ return this->do_grouping(); }
string_type
truename() const
{ return this->do_truename(); }
string_type
falsename() const
{ return this->do_falsename(); }
protected:
virtual
~numpunct();
virtual char_type
do_decimal_point() const
{ return _M_data->_M_decimal_point; }
virtual char_type
do_thousands_sep() const
{ return _M_data->_M_thousands_sep; }
virtual string
do_grouping() const
{ return _M_data->_M_grouping; }
virtual string_type
do_truename() const
{ return _M_data->_M_truename; }
virtual string_type
do_falsename() const
{ return _M_data->_M_falsename; }
void
_M_initialize_numpunct(__c_locale __cloc = 0);
};
template<typename _CharT>
locale::id numpunct<_CharT>::id;
template<>
numpunct<char>::~numpunct();
template<>
void
numpunct<char>::_M_initialize_numpunct(__c_locale __cloc);
template<>
numpunct<wchar_t>::~numpunct();
template<>
void
numpunct<wchar_t>::_M_initialize_numpunct(__c_locale __cloc);
template<typename _CharT>
class numpunct_byname : public numpunct<_CharT>
{
public:
typedef _CharT char_type;
typedef basic_string<_CharT> string_type;
explicit
numpunct_byname(const char* __s, size_t __refs = 0)
: numpunct<_CharT>(__refs)
{
if (__builtin_strcmp(__s, "C") != 0
&& __builtin_strcmp(__s, "POSIX") != 0)
{
__c_locale __tmp;
this->_S_create_c_locale(__tmp, __s);
this->_M_initialize_numpunct(__tmp);
this->_S_destroy_c_locale(__tmp);
}
}
explicit
numpunct_byname(const string& __s, size_t __refs = 0)
: numpunct_byname(__s.c_str(), __refs) { }
protected:
virtual
~numpunct_byname() { }
};
}
template<typename _CharT, typename _InIter>
class num_get : public locale::facet
{
public:
typedef _CharT char_type;
typedef _InIter iter_type;
static locale::id id;
explicit
num_get(size_t __refs = 0) : facet(__refs) { }
iter_type
get(iter_type __in, iter_type __end, ios_base& __io,
ios_base::iostate& __err, bool& __v) const
{ return this->do_get(__in, __end, __io, __err, __v); }
iter_type
get(iter_type __in, iter_type __end, ios_base& __io,
ios_base::iostate& __err, long& __v) const
{ return this->do_get(__in, __end, __io, __err, __v); }
iter_type
get(iter_type __in, iter_type __end, ios_base& __io,
ios_base::iostate& __err, unsigned short& __v) const
{ return this->do_get(__in, __end, __io, __err, __v); }
iter_type
get(iter_type __in, iter_type __end, ios_base& __io,
ios_base::iostate& __err, unsigned int& __v) const
{ return this->do_get(__in, __end, __io, __err, __v); }
iter_type
get(iter_type __in, iter_type __end, ios_base& __io,
ios_base::iostate& __err, unsigned long& __v) const
{ return this->do_get(__in, __end, __io, __err, __v); }
iter_type
get(iter_type __in, iter_type __end, ios_base& __io,
ios_base::iostate& __err, long long& __v) const
{ return this->do_get(__in, __end, __io, __err, __v); }
iter_type
get(iter_type __in, iter_type __end, ios_base& __io,
ios_base::iostate& __err, unsigned long long& __v) const
{ return this->do_get(__in, __end, __io, __err, __v); }
iter_type
get(iter_type __in, iter_type __end, ios_base& __io,
ios_base::iostate& __err, float& __v) const
{ return this->do_get(__in, __end, __io, __err, __v); }
iter_type
get(iter_type __in, iter_type __end, ios_base& __io,
ios_base::iostate& __err, double& __v) const
{ return this->do_get(__in, __end, __io, __err, __v); }
iter_type
get(iter_type __in, iter_type __end, ios_base& __io,
ios_base::iostate& __err, long double& __v) const
{ return this->do_get(__in, __end, __io, __err, __v); }
iter_type
get(iter_type __in, iter_type __end, ios_base& __io,
ios_base::iostate& __err, void*& __v) const
{ return this->do_get(__in, __end, __io, __err, __v); }
protected:
virtual ~num_get() { }
__attribute ((__abi_tag__ ("cxx11")))
iter_type
_M_extract_float(iter_type, iter_type, ios_base&, ios_base::iostate&,
string&) const;
template<typename _ValueT>
__attribute ((__abi_tag__ ("cxx11")))
iter_type
_M_extract_int(iter_type, iter_type, ios_base&, ios_base::iostate&,
_ValueT&) const;
template<typename _CharT2>
typename __gnu_cxx::__enable_if<__is_char<_CharT2>::__value, int>::__type
_M_find(const _CharT2*, size_t __len, _CharT2 __c) const
{
int __ret = -1;
if (__len <= 10)
{
if (__c >= _CharT2('0') && __c < _CharT2(_CharT2('0') + __len))
__ret = __c - _CharT2('0');
}
else
{
if (__c >= _CharT2('0') && __c <= _CharT2('9'))
__ret = __c - _CharT2('0');
else if (__c >= _CharT2('a') && __c <= _CharT2('f'))
__ret = 10 + (__c - _CharT2('a'));
else if (__c >= _CharT2('A') && __c <= _CharT2('F'))
__ret = 10 + (__c - _CharT2('A'));
}
return __ret;
}
template<typename _CharT2>
typename __gnu_cxx::__enable_if<!__is_char<_CharT2>::__value,
int>::__type
_M_find(const _CharT2* __zero, size_t __len, _CharT2 __c) const
{
int __ret = -1;
const char_type* __q = char_traits<_CharT2>::find(__zero, __len, __c);
if (__q)
{
__ret = __q - __zero;
if (__ret > 15)
__ret -= 6;
}
return __ret;
}
virtual iter_type
do_get(iter_type, iter_type, ios_base&, ios_base::iostate&, bool&) const;
virtual iter_type
do_get(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, long& __v) const
{ return _M_extract_int(__beg, __end, __io, __err, __v); }
virtual iter_type
do_get(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, unsigned short& __v) const
{ return _M_extract_int(__beg, __end, __io, __err, __v); }
virtual iter_type
do_get(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, unsigned int& __v) const
{ return _M_extract_int(__beg, __end, __io, __err, __v); }
virtual iter_type
do_get(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, unsigned long& __v) const
{ return _M_extract_int(__beg, __end, __io, __err, __v); }
virtual iter_type
do_get(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, long long& __v) const
{ return _M_extract_int(__beg, __end, __io, __err, __v); }
virtual iter_type
do_get(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, unsigned long long& __v) const
{ return _M_extract_int(__beg, __end, __io, __err, __v); }
virtual iter_type
do_get(iter_type, iter_type, ios_base&, ios_base::iostate&, float&) const;
virtual iter_type
do_get(iter_type, iter_type, ios_base&, ios_base::iostate&,
double&) const;
virtual iter_type
do_get(iter_type, iter_type, ios_base&, ios_base::iostate&,
long double&) const;
virtual iter_type
do_get(iter_type, iter_type, ios_base&, ios_base::iostate&, void*&) const;
};
template<typename _CharT, typename _InIter>
locale::id num_get<_CharT, _InIter>::id;
template<typename _CharT, typename _OutIter>
class num_put : public locale::facet
{
public:
typedef _CharT char_type;
typedef _OutIter iter_type;
static locale::id id;
explicit
num_put(size_t __refs = 0) : facet(__refs) { }
iter_type
put(iter_type __s, ios_base& __io, char_type __fill, bool __v) const
{ return this->do_put(__s, __io, __fill, __v); }
iter_type
put(iter_type __s, ios_base& __io, char_type __fill, long __v) const
{ return this->do_put(__s, __io, __fill, __v); }
iter_type
put(iter_type __s, ios_base& __io, char_type __fill,
unsigned long __v) const
{ return this->do_put(__s, __io, __fill, __v); }
iter_type
put(iter_type __s, ios_base& __io, char_type __fill, long long __v) const
{ return this->do_put(__s, __io, __fill, __v); }
iter_type
put(iter_type __s, ios_base& __io, char_type __fill,
unsigned long long __v) const
{ return this->do_put(__s, __io, __fill, __v); }
iter_type
put(iter_type __s, ios_base& __io, char_type __fill, double __v) const
{ return this->do_put(__s, __io, __fill, __v); }
iter_type
put(iter_type __s, ios_base& __io, char_type __fill,
long double __v) const
{ return this->do_put(__s, __io, __fill, __v); }
iter_type
put(iter_type __s, ios_base& __io, char_type __fill,
const void* __v) const
{ return this->do_put(__s, __io, __fill, __v); }
protected:
template<typename _ValueT>
iter_type
_M_insert_float(iter_type, ios_base& __io, char_type __fill,
char __mod, _ValueT __v) const;
void
_M_group_float(const char* __grouping, size_t __grouping_size,
char_type __sep, const char_type* __p, char_type* __new,
char_type* __cs, int& __len) const;
template<typename _ValueT>
iter_type
_M_insert_int(iter_type, ios_base& __io, char_type __fill,
_ValueT __v) const;
void
_M_group_int(const char* __grouping, size_t __grouping_size,
char_type __sep, ios_base& __io, char_type* __new,
char_type* __cs, int& __len) const;
void
_M_pad(char_type __fill, streamsize __w, ios_base& __io,
char_type* __new, const char_type* __cs, int& __len) const;
virtual
~num_put() { }
virtual iter_type
do_put(iter_type __s, ios_base& __io, char_type __fill, bool __v) const;
virtual iter_type
do_put(iter_type __s, ios_base& __io, char_type __fill, long __v) const
{ return _M_insert_int(__s, __io, __fill, __v); }
virtual iter_type
do_put(iter_type __s, ios_base& __io, char_type __fill,
unsigned long __v) const
{ return _M_insert_int(__s, __io, __fill, __v); }
virtual iter_type
do_put(iter_type __s, ios_base& __io, char_type __fill,
long long __v) const
{ return _M_insert_int(__s, __io, __fill, __v); }
virtual iter_type
do_put(iter_type __s, ios_base& __io, char_type __fill,
unsigned long long __v) const
{ return _M_insert_int(__s, __io, __fill, __v); }
virtual iter_type
do_put(iter_type, ios_base&, char_type, double) const;
virtual iter_type
do_put(iter_type, ios_base&, char_type, long double) const;
virtual iter_type
do_put(iter_type, ios_base&, char_type, const void*) const;
};
template <typename _CharT, typename _OutIter>
locale::id num_put<_CharT, _OutIter>::id;
template<typename _CharT>
inline bool
isspace(_CharT __c, const locale& __loc)
{ return use_facet<ctype<_CharT> >(__loc).is(ctype_base::space, __c); }
template<typename _CharT>
inline bool
isprint(_CharT __c, const locale& __loc)
{ return use_facet<ctype<_CharT> >(__loc).is(ctype_base::print, __c); }
template<typename _CharT>
inline bool
iscntrl(_CharT __c, const locale& __loc)
{ return use_facet<ctype<_CharT> >(__loc).is(ctype_base::cntrl, __c); }
template<typename _CharT>
inline bool
isupper(_CharT __c, const locale& __loc)
{ return use_facet<ctype<_CharT> >(__loc).is(ctype_base::upper, __c); }
template<typename _CharT>
inline bool
islower(_CharT __c, const locale& __loc)
{ return use_facet<ctype<_CharT> >(__loc).is(ctype_base::lower, __c); }
template<typename _CharT>
inline bool
isalpha(_CharT __c, const locale& __loc)
{ return use_facet<ctype<_CharT> >(__loc).is(ctype_base::alpha, __c); }
template<typename _CharT>
inline bool
isdigit(_CharT __c, const locale& __loc)
{ return use_facet<ctype<_CharT> >(__loc).is(ctype_base::digit, __c); }
template<typename _CharT>
inline bool
ispunct(_CharT __c, const locale& __loc)
{ return use_facet<ctype<_CharT> >(__loc).is(ctype_base::punct, __c); }
template<typename _CharT>
inline bool
isxdigit(_CharT __c, const locale& __loc)
{ return use_facet<ctype<_CharT> >(__loc).is(ctype_base::xdigit, __c); }
template<typename _CharT>
inline bool
isalnum(_CharT __c, const locale& __loc)
{ return use_facet<ctype<_CharT> >(__loc).is(ctype_base::alnum, __c); }
template<typename _CharT>
inline bool
isgraph(_CharT __c, const locale& __loc)
{ return use_facet<ctype<_CharT> >(__loc).is(ctype_base::graph, __c); }
template<typename _CharT>
inline bool
isblank(_CharT __c, const locale& __loc)
{ return use_facet<ctype<_CharT> >(__loc).is(ctype_base::blank, __c); }
template<typename _CharT>
inline _CharT
toupper(_CharT __c, const locale& __loc)
{ return use_facet<ctype<_CharT> >(__loc).toupper(__c); }
template<typename _CharT>
inline _CharT
tolower(_CharT __c, const locale& __loc)
{ return use_facet<ctype<_CharT> >(__loc).tolower(__c); }
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Facet>
struct __use_cache
{
const _Facet*
operator() (const locale& __loc) const;
};
template<typename _CharT>
struct __use_cache<__numpunct_cache<_CharT> >
{
const __numpunct_cache<_CharT>*
operator() (const locale& __loc) const
{
const size_t __i = numpunct<_CharT>::id._M_id();
const locale::facet** __caches = __loc._M_impl->_M_caches;
if (!__caches[__i])
{
__numpunct_cache<_CharT>* __tmp = 0;
try
{
__tmp = new __numpunct_cache<_CharT>;
__tmp->_M_cache(__loc);
}
catch(...)
{
delete __tmp;
throw;
}
__loc._M_impl->_M_install_cache(__tmp, __i);
}
return static_cast<const __numpunct_cache<_CharT>*>(__caches[__i]);
}
};
template<typename _CharT>
void
__numpunct_cache<_CharT>::_M_cache(const locale& __loc)
{
const numpunct<_CharT>& __np = use_facet<numpunct<_CharT> >(__loc);
char* __grouping = 0;
_CharT* __truename = 0;
_CharT* __falsename = 0;
try
{
const string& __g = __np.grouping();
_M_grouping_size = __g.size();
__grouping = new char[_M_grouping_size];
__g.copy(__grouping, _M_grouping_size);
_M_use_grouping = (_M_grouping_size
&& static_cast<signed char>(__grouping[0]) > 0
&& (__grouping[0]
!= __gnu_cxx::__numeric_traits<char>::__max));
const basic_string<_CharT>& __tn = __np.truename();
_M_truename_size = __tn.size();
__truename = new _CharT[_M_truename_size];
__tn.copy(__truename, _M_truename_size);
const basic_string<_CharT>& __fn = __np.falsename();
_M_falsename_size = __fn.size();
__falsename = new _CharT[_M_falsename_size];
__fn.copy(__falsename, _M_falsename_size);
_M_decimal_point = __np.decimal_point();
_M_thousands_sep = __np.thousands_sep();
const ctype<_CharT>& __ct = use_facet<ctype<_CharT> >(__loc);
__ct.widen(__num_base::_S_atoms_out,
__num_base::_S_atoms_out
+ __num_base::_S_oend, _M_atoms_out);
__ct.widen(__num_base::_S_atoms_in,
__num_base::_S_atoms_in
+ __num_base::_S_iend, _M_atoms_in);
_M_grouping = __grouping;
_M_truename = __truename;
_M_falsename = __falsename;
_M_allocated = true;
}
catch(...)
{
delete [] __grouping;
delete [] __truename;
delete [] __falsename;
throw;
}
}
__attribute__ ((__pure__)) bool
__verify_grouping(const char* __grouping, size_t __grouping_size,
const string& __grouping_tmp) throw ();
template<typename _CharT, typename _InIter>
__attribute ((__abi_tag__ ("cxx11")))
_InIter
num_get<_CharT, _InIter>::
_M_extract_float(_InIter __beg, _InIter __end, ios_base& __io,
ios_base::iostate& __err, string& __xtrc) const
{
typedef char_traits<_CharT> __traits_type;
typedef __numpunct_cache<_CharT> __cache_type;
__use_cache<__cache_type> __uc;
const locale& __loc = __io._M_getloc();
const __cache_type* __lc = __uc(__loc);
const _CharT* __lit = __lc->_M_atoms_in;
char_type __c = char_type();
bool __testeof = __beg == __end;
if (!__testeof)
{
__c = *__beg;
const bool __plus = __c == __lit[__num_base::_S_iplus];
if ((__plus || __c == __lit[__num_base::_S_iminus])
&& !(__lc->_M_use_grouping && __c == __lc->_M_thousands_sep)
&& !(__c == __lc->_M_decimal_point))
{
__xtrc += __plus ? '+' : '-';
if (++__beg != __end)
__c = *__beg;
else
__testeof = true;
}
}
bool __found_mantissa = false;
int __sep_pos = 0;
while (!__testeof)
{
if ((__lc->_M_use_grouping && __c == __lc->_M_thousands_sep)
|| __c == __lc->_M_decimal_point)
break;
else if (__c == __lit[__num_base::_S_izero])
{
if (!__found_mantissa)
{
__xtrc += '0';
__found_mantissa = true;
}
++__sep_pos;
if (++__beg != __end)
__c = *__beg;
else
__testeof = true;
}
else
break;
}
bool __found_dec = false;
bool __found_sci = false;
string __found_grouping;
if (__lc->_M_use_grouping)
__found_grouping.reserve(32);
const char_type* __lit_zero = __lit + __num_base::_S_izero;
if (!__lc->_M_allocated)
while (!__testeof)
{
const int __digit = _M_find(__lit_zero, 10, __c);
if (__digit != -1)
{
__xtrc += '0' + __digit;
__found_mantissa = true;
}
else if (__c == __lc->_M_decimal_point
&& !__found_dec && !__found_sci)
{
__xtrc += '.';
__found_dec = true;
}
else if ((__c == __lit[__num_base::_S_ie]
|| __c == __lit[__num_base::_S_iE])
&& !__found_sci && __found_mantissa)
{
__xtrc += 'e';
__found_sci = true;
if (++__beg != __end)
{
__c = *__beg;
const bool __plus = __c == __lit[__num_base::_S_iplus];
if (__plus || __c == __lit[__num_base::_S_iminus])
__xtrc += __plus ? '+' : '-';
else
continue;
}
else
{
__testeof = true;
break;
}
}
else
break;
if (++__beg != __end)
__c = *__beg;
else
__testeof = true;
}
else
while (!__testeof)
{
if (__lc->_M_use_grouping && __c == __lc->_M_thousands_sep)
{
if (!__found_dec && !__found_sci)
{
if (__sep_pos)
{
__found_grouping += static_cast<char>(__sep_pos);
__sep_pos = 0;
}
else
{
__xtrc.clear();
break;
}
}
else
break;
}
else if (__c == __lc->_M_decimal_point)
{
if (!__found_dec && !__found_sci)
{
if (__found_grouping.size())
__found_grouping += static_cast<char>(__sep_pos);
__xtrc += '.';
__found_dec = true;
}
else
break;
}
else
{
const char_type* __q =
__traits_type::find(__lit_zero, 10, __c);
if (__q)
{
__xtrc += '0' + (__q - __lit_zero);
__found_mantissa = true;
++__sep_pos;
}
else if ((__c == __lit[__num_base::_S_ie]
|| __c == __lit[__num_base::_S_iE])
&& !__found_sci && __found_mantissa)
{
if (__found_grouping.size() && !__found_dec)
__found_grouping += static_cast<char>(__sep_pos);
__xtrc += 'e';
__found_sci = true;
if (++__beg != __end)
{
__c = *__beg;
const bool __plus = __c == __lit[__num_base::_S_iplus];
if ((__plus || __c == __lit[__num_base::_S_iminus])
&& !(__lc->_M_use_grouping
&& __c == __lc->_M_thousands_sep)
&& !(__c == __lc->_M_decimal_point))
__xtrc += __plus ? '+' : '-';
else
continue;
}
else
{
__testeof = true;
break;
}
}
else
break;
}
if (++__beg != __end)
__c = *__beg;
else
__testeof = true;
}
if (__found_grouping.size())
{
if (!__found_dec && !__found_sci)
__found_grouping += static_cast<char>(__sep_pos);
if (!std::__verify_grouping(__lc->_M_grouping,
__lc->_M_grouping_size,
__found_grouping))
__err = ios_base::failbit;
}
return __beg;
}
template<typename _CharT, typename _InIter>
template<typename _ValueT>
__attribute ((__abi_tag__ ("cxx11")))
_InIter
num_get<_CharT, _InIter>::
_M_extract_int(_InIter __beg, _InIter __end, ios_base& __io,
ios_base::iostate& __err, _ValueT& __v) const
{
typedef char_traits<_CharT> __traits_type;
using __gnu_cxx::__add_unsigned;
typedef typename __add_unsigned<_ValueT>::__type __unsigned_type;
typedef __numpunct_cache<_CharT> __cache_type;
__use_cache<__cache_type> __uc;
const locale& __loc = __io._M_getloc();
const __cache_type* __lc = __uc(__loc);
const _CharT* __lit = __lc->_M_atoms_in;
char_type __c = char_type();
const ios_base::fmtflags __basefield = __io.flags()
& ios_base::basefield;
const bool __oct = __basefield == ios_base::oct;
int __base = __oct ? 8 : (__basefield == ios_base::hex ? 16 : 10);
bool __testeof = __beg == __end;
bool __negative = false;
if (!__testeof)
{
__c = *__beg;
__negative = __c == __lit[__num_base::_S_iminus];
if ((__negative || __c == __lit[__num_base::_S_iplus])
&& !(__lc->_M_use_grouping && __c == __lc->_M_thousands_sep)
&& !(__c == __lc->_M_decimal_point))
{
if (++__beg != __end)
__c = *__beg;
else
__testeof = true;
}
}
bool __found_zero = false;
int __sep_pos = 0;
while (!__testeof)
{
if ((__lc->_M_use_grouping && __c == __lc->_M_thousands_sep)
|| __c == __lc->_M_decimal_point)
break;
else if (__c == __lit[__num_base::_S_izero]
&& (!__found_zero || __base == 10))
{
__found_zero = true;
++__sep_pos;
if (__basefield == 0)
__base = 8;
if (__base == 8)
__sep_pos = 0;
}
else if (__found_zero
&& (__c == __lit[__num_base::_S_ix]
|| __c == __lit[__num_base::_S_iX]))
{
if (__basefield == 0)
__base = 16;
if (__base == 16)
{
__found_zero = false;
__sep_pos = 0;
}
else
break;
}
else
break;
if (++__beg != __end)
{
__c = *__beg;
if (!__found_zero)
break;
}
else
__testeof = true;
}
const size_t __len = (__base == 16 ? __num_base::_S_iend
- __num_base::_S_izero : __base);
typedef __gnu_cxx::__numeric_traits<_ValueT> __num_traits;
string __found_grouping;
if (__lc->_M_use_grouping)
__found_grouping.reserve(32);
bool __testfail = false;
bool __testoverflow = false;
const __unsigned_type __max =
(__negative && __num_traits::__is_signed)
? -static_cast<__unsigned_type>(__num_traits::__min)
: __num_traits::__max;
const __unsigned_type __smax = __max / __base;
__unsigned_type __result = 0;
int __digit = 0;
const char_type* __lit_zero = __lit + __num_base::_S_izero;
if (!__lc->_M_allocated)
while (!__testeof)
{
__digit = _M_find(__lit_zero, __len, __c);
if (__digit == -1)
break;
if (__result > __smax)
__testoverflow = true;
else
{
__result *= __base;
__testoverflow |= __result > __max - __digit;
__result += __digit;
++__sep_pos;
}
if (++__beg != __end)
__c = *__beg;
else
__testeof = true;
}
else
while (!__testeof)
{
if (__lc->_M_use_grouping && __c == __lc->_M_thousands_sep)
{
if (__sep_pos)
{
__found_grouping += static_cast<char>(__sep_pos);
__sep_pos = 0;
}
else
{
__testfail = true;
break;
}
}
else if (__c == __lc->_M_decimal_point)
break;
else
{
const char_type* __q =
__traits_type::find(__lit_zero, __len, __c);
if (!__q)
break;
__digit = __q - __lit_zero;
if (__digit > 15)
__digit -= 6;
if (__result > __smax)
__testoverflow = true;
else
{
__result *= __base;
__testoverflow |= __result > __max - __digit;
__result += __digit;
++__sep_pos;
}
}
if (++__beg != __end)
__c = *__beg;
else
__testeof = true;
}
if (__found_grouping.size())
{
__found_grouping += static_cast<char>(__sep_pos);
if (!std::__verify_grouping(__lc->_M_grouping,
__lc->_M_grouping_size,
__found_grouping))
__err = ios_base::failbit;
}
if ((!__sep_pos && !__found_zero && !__found_grouping.size())
|| __testfail)
{
__v = 0;
__err = ios_base::failbit;
}
else if (__testoverflow)
{
if (__negative && __num_traits::__is_signed)
__v = __num_traits::__min;
else
__v = __num_traits::__max;
__err = ios_base::failbit;
}
else
__v = __negative ? -__result : __result;
if (__testeof)
__err |= ios_base::eofbit;
return __beg;
}
template<typename _CharT, typename _InIter>
_InIter
num_get<_CharT, _InIter>::
do_get(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, bool& __v) const
{
if (!(__io.flags() & ios_base::boolalpha))
{
long __l = -1;
__beg = _M_extract_int(__beg, __end, __io, __err, __l);
if (__l == 0 || __l == 1)
__v = bool(__l);
else
{
__v = true;
__err = ios_base::failbit;
if (__beg == __end)
__err |= ios_base::eofbit;
}
}
else
{
typedef __numpunct_cache<_CharT> __cache_type;
__use_cache<__cache_type> __uc;
const locale& __loc = __io._M_getloc();
const __cache_type* __lc = __uc(__loc);
bool __testf = true;
bool __testt = true;
bool __donef = __lc->_M_falsename_size == 0;
bool __donet = __lc->_M_truename_size == 0;
bool __testeof = false;
size_t __n = 0;
while (!__donef || !__donet)
{
if (__beg == __end)
{
__testeof = true;
break;
}
const char_type __c = *__beg;
if (!__donef)
__testf = __c == __lc->_M_falsename[__n];
if (!__testf && __donet)
break;
if (!__donet)
__testt = __c == __lc->_M_truename[__n];
if (!__testt && __donef)
break;
if (!__testt && !__testf)
break;
++__n;
++__beg;
__donef = !__testf || __n >= __lc->_M_falsename_size;
__donet = !__testt || __n >= __lc->_M_truename_size;
}
if (__testf && __n == __lc->_M_falsename_size && __n)
{
__v = false;
if (__testt && __n == __lc->_M_truename_size)
__err = ios_base::failbit;
else
__err = __testeof ? ios_base::eofbit : ios_base::goodbit;
}
else if (__testt && __n == __lc->_M_truename_size && __n)
{
__v = true;
__err = __testeof ? ios_base::eofbit : ios_base::goodbit;
}
else
{
__v = false;
__err = ios_base::failbit;
if (__testeof)
__err |= ios_base::eofbit;
}
}
return __beg;
}
template<typename _CharT, typename _InIter>
_InIter
num_get<_CharT, _InIter>::
do_get(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, float& __v) const
{
string __xtrc;
__xtrc.reserve(32);
__beg = _M_extract_float(__beg, __end, __io, __err, __xtrc);
std::__convert_to_v(__xtrc.c_str(), __v, __err, _S_get_c_locale());
if (__beg == __end)
__err |= ios_base::eofbit;
return __beg;
}
template<typename _CharT, typename _InIter>
_InIter
num_get<_CharT, _InIter>::
do_get(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, double& __v) const
{
string __xtrc;
__xtrc.reserve(32);
__beg = _M_extract_float(__beg, __end, __io, __err, __xtrc);
std::__convert_to_v(__xtrc.c_str(), __v, __err, _S_get_c_locale());
if (__beg == __end)
__err |= ios_base::eofbit;
return __beg;
}
template<typename _CharT, typename _InIter>
_InIter
num_get<_CharT, _InIter>::
do_get(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, long double& __v) const
{
string __xtrc;
__xtrc.reserve(32);
__beg = _M_extract_float(__beg, __end, __io, __err, __xtrc);
std::__convert_to_v(__xtrc.c_str(), __v, __err, _S_get_c_locale());
if (__beg == __end)
__err |= ios_base::eofbit;
return __beg;
}
template<typename _CharT, typename _InIter>
_InIter
num_get<_CharT, _InIter>::
do_get(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, void*& __v) const
{
typedef ios_base::fmtflags fmtflags;
const fmtflags __fmt = __io.flags();
__io.flags((__fmt & ~ios_base::basefield) | ios_base::hex);
typedef __gnu_cxx::__conditional_type<(sizeof(void*)
<= sizeof(unsigned long)),
unsigned long, unsigned long long>::__type _UIntPtrType;
_UIntPtrType __ul;
__beg = _M_extract_int(__beg, __end, __io, __err, __ul);
__io.flags(__fmt);
__v = reinterpret_cast<void*>(__ul);
return __beg;
}
template<typename _CharT, typename _OutIter>
void
num_put<_CharT, _OutIter>::
_M_pad(_CharT __fill, streamsize __w, ios_base& __io,
_CharT* __new, const _CharT* __cs, int& __len) const
{
__pad<_CharT, char_traits<_CharT> >::_S_pad(__io, __fill, __new,
__cs, __w, __len);
__len = static_cast<int>(__w);
}
template<typename _CharT, typename _ValueT>
int
__int_to_char(_CharT* __bufend, _ValueT __v, const _CharT* __lit,
ios_base::fmtflags __flags, bool __dec)
{
_CharT* __buf = __bufend;
if (__builtin_expect(__dec, true))
{
do
{
*--__buf = __lit[(__v % 10) + __num_base::_S_odigits];
__v /= 10;
}
while (__v != 0);
}
else if ((__flags & ios_base::basefield) == ios_base::oct)
{
do
{
*--__buf = __lit[(__v & 0x7) + __num_base::_S_odigits];
__v >>= 3;
}
while (__v != 0);
}
else
{
const bool __uppercase = __flags & ios_base::uppercase;
const int __case_offset = __uppercase ? __num_base::_S_oudigits
: __num_base::_S_odigits;
do
{
*--__buf = __lit[(__v & 0xf) + __case_offset];
__v >>= 4;
}
while (__v != 0);
}
return __bufend - __buf;
}
template<typename _CharT, typename _OutIter>
void
num_put<_CharT, _OutIter>::
_M_group_int(const char* __grouping, size_t __grouping_size, _CharT __sep,
ios_base&, _CharT* __new, _CharT* __cs, int& __len) const
{
_CharT* __p = std::__add_grouping(__new, __sep, __grouping,
__grouping_size, __cs, __cs + __len);
__len = __p - __new;
}
template<typename _CharT, typename _OutIter>
template<typename _ValueT>
_OutIter
num_put<_CharT, _OutIter>::
_M_insert_int(_OutIter __s, ios_base& __io, _CharT __fill,
_ValueT __v) const
{
using __gnu_cxx::__add_unsigned;
typedef typename __add_unsigned<_ValueT>::__type __unsigned_type;
typedef __numpunct_cache<_CharT> __cache_type;
__use_cache<__cache_type> __uc;
const locale& __loc = __io._M_getloc();
const __cache_type* __lc = __uc(__loc);
const _CharT* __lit = __lc->_M_atoms_out;
const ios_base::fmtflags __flags = __io.flags();
const int __ilen = 5 * sizeof(_ValueT);
_CharT* __cs = static_cast<_CharT*>(__builtin_alloca(sizeof(_CharT)
* __ilen));
const ios_base::fmtflags __basefield = __flags & ios_base::basefield;
const bool __dec = (__basefield != ios_base::oct
&& __basefield != ios_base::hex);
const __unsigned_type __u = ((__v > 0 || !__dec)
? __unsigned_type(__v)
: -__unsigned_type(__v));
int __len = __int_to_char(__cs + __ilen, __u, __lit, __flags, __dec);
__cs += __ilen - __len;
if (__lc->_M_use_grouping)
{
_CharT* __cs2 = static_cast<_CharT*>(__builtin_alloca(sizeof(_CharT)
* (__len + 1)
* 2));
_M_group_int(__lc->_M_grouping, __lc->_M_grouping_size,
__lc->_M_thousands_sep, __io, __cs2 + 2, __cs, __len);
__cs = __cs2 + 2;
}
if (__builtin_expect(__dec, true))
{
if (__v >= 0)
{
if (bool(__flags & ios_base::showpos)
&& __gnu_cxx::__numeric_traits<_ValueT>::__is_signed)
*--__cs = __lit[__num_base::_S_oplus], ++__len;
}
else
*--__cs = __lit[__num_base::_S_ominus], ++__len;
}
else if (bool(__flags & ios_base::showbase) && __v)
{
if (__basefield == ios_base::oct)
*--__cs = __lit[__num_base::_S_odigits], ++__len;
else
{
const bool __uppercase = __flags & ios_base::uppercase;
*--__cs = __lit[__num_base::_S_ox + __uppercase];
*--__cs = __lit[__num_base::_S_odigits];
__len += 2;
}
}
const streamsize __w = __io.width();
if (__w > static_cast<streamsize>(__len))
{
_CharT* __cs3 = static_cast<_CharT*>(__builtin_alloca(sizeof(_CharT)
* __w));
_M_pad(__fill, __w, __io, __cs3, __cs, __len);
__cs = __cs3;
}
__io.width(0);
return std::__write(__s, __cs, __len);
}
template<typename _CharT, typename _OutIter>
void
num_put<_CharT, _OutIter>::
_M_group_float(const char* __grouping, size_t __grouping_size,
_CharT __sep, const _CharT* __p, _CharT* __new,
_CharT* __cs, int& __len) const
{
const int __declen = __p ? __p - __cs : __len;
_CharT* __p2 = std::__add_grouping(__new, __sep, __grouping,
__grouping_size,
__cs, __cs + __declen);
int __newlen = __p2 - __new;
if (__p)
{
char_traits<_CharT>::copy(__p2, __p, __len - __declen);
__newlen += __len - __declen;
}
__len = __newlen;
}
template<typename _CharT, typename _OutIter>
template<typename _ValueT>
_OutIter
num_put<_CharT, _OutIter>::
_M_insert_float(_OutIter __s, ios_base& __io, _CharT __fill, char __mod,
_ValueT __v) const
{
typedef __numpunct_cache<_CharT> __cache_type;
__use_cache<__cache_type> __uc;
const locale& __loc = __io._M_getloc();
const __cache_type* __lc = __uc(__loc);
const streamsize __prec = __io.precision() < 0 ? 6 : __io.precision();
const int __max_digits =
__gnu_cxx::__numeric_traits<_ValueT>::__digits10;
int __len;
char __fbuf[16];
__num_base::_S_format_float(__io, __fbuf, __mod);
const bool __use_prec =
(__io.flags() & ios_base::floatfield) != ios_base::floatfield;
int __cs_size = __max_digits * 3;
char* __cs = static_cast<char*>(__builtin_alloca(__cs_size));
if (__use_prec)
__len = std::__convert_from_v(_S_get_c_locale(), __cs, __cs_size,
__fbuf, __prec, __v);
else
__len = std::__convert_from_v(_S_get_c_locale(), __cs, __cs_size,
__fbuf, __v);
if (__len >= __cs_size)
{
__cs_size = __len + 1;
__cs = static_cast<char*>(__builtin_alloca(__cs_size));
if (__use_prec)
__len = std::__convert_from_v(_S_get_c_locale(), __cs, __cs_size,
__fbuf, __prec, __v);
else
__len = std::__convert_from_v(_S_get_c_locale(), __cs, __cs_size,
__fbuf, __v);
}
const ctype<_CharT>& __ctype = use_facet<ctype<_CharT> >(__loc);
_CharT* __ws = static_cast<_CharT*>(__builtin_alloca(sizeof(_CharT)
* __len));
__ctype.widen(__cs, __cs + __len, __ws);
_CharT* __wp = 0;
const char* __p = char_traits<char>::find(__cs, __len, '.');
if (__p)
{
__wp = __ws + (__p - __cs);
*__wp = __lc->_M_decimal_point;
}
if (__lc->_M_use_grouping
&& (__wp || __len < 3 || (__cs[1] <= '9' && __cs[2] <= '9'
&& __cs[1] >= '0' && __cs[2] >= '0')))
{
_CharT* __ws2 = static_cast<_CharT*>(__builtin_alloca(sizeof(_CharT)
* __len * 2));
streamsize __off = 0;
if (__cs[0] == '-' || __cs[0] == '+')
{
__off = 1;
__ws2[0] = __ws[0];
__len -= 1;
}
_M_group_float(__lc->_M_grouping, __lc->_M_grouping_size,
__lc->_M_thousands_sep, __wp, __ws2 + __off,
__ws + __off, __len);
__len += __off;
__ws = __ws2;
}
const streamsize __w = __io.width();
if (__w > static_cast<streamsize>(__len))
{
_CharT* __ws3 = static_cast<_CharT*>(__builtin_alloca(sizeof(_CharT)
* __w));
_M_pad(__fill, __w, __io, __ws3, __ws, __len);
__ws = __ws3;
}
__io.width(0);
return std::__write(__s, __ws, __len);
}
template<typename _CharT, typename _OutIter>
_OutIter
num_put<_CharT, _OutIter>::
do_put(iter_type __s, ios_base& __io, char_type __fill, bool __v) const
{
const ios_base::fmtflags __flags = __io.flags();
if ((__flags & ios_base::boolalpha) == 0)
{
const long __l = __v;
__s = _M_insert_int(__s, __io, __fill, __l);
}
else
{
typedef __numpunct_cache<_CharT> __cache_type;
__use_cache<__cache_type> __uc;
const locale& __loc = __io._M_getloc();
const __cache_type* __lc = __uc(__loc);
const _CharT* __name = __v ? __lc->_M_truename
: __lc->_M_falsename;
int __len = __v ? __lc->_M_truename_size
: __lc->_M_falsename_size;
const streamsize __w = __io.width();
if (__w > static_cast<streamsize>(__len))
{
const streamsize __plen = __w - __len;
_CharT* __ps
= static_cast<_CharT*>(__builtin_alloca(sizeof(_CharT)
* __plen));
char_traits<_CharT>::assign(__ps, __plen, __fill);
__io.width(0);
if ((__flags & ios_base::adjustfield) == ios_base::left)
{
__s = std::__write(__s, __name, __len);
__s = std::__write(__s, __ps, __plen);
}
else
{
__s = std::__write(__s, __ps, __plen);
__s = std::__write(__s, __name, __len);
}
return __s;
}
__io.width(0);
__s = std::__write(__s, __name, __len);
}
return __s;
}
template<typename _CharT, typename _OutIter>
_OutIter
num_put<_CharT, _OutIter>::
do_put(iter_type __s, ios_base& __io, char_type __fill, double __v) const
{ return _M_insert_float(__s, __io, __fill, char(), __v); }
template<typename _CharT, typename _OutIter>
_OutIter
num_put<_CharT, _OutIter>::
do_put(iter_type __s, ios_base& __io, char_type __fill,
long double __v) const
{ return _M_insert_float(__s, __io, __fill, 'L', __v); }
template<typename _CharT, typename _OutIter>
_OutIter
num_put<_CharT, _OutIter>::
do_put(iter_type __s, ios_base& __io, char_type __fill,
const void* __v) const
{
const ios_base::fmtflags __flags = __io.flags();
const ios_base::fmtflags __fmt = ~(ios_base::basefield
| ios_base::uppercase);
__io.flags((__flags & __fmt) | (ios_base::hex | ios_base::showbase));
typedef __gnu_cxx::__conditional_type<(sizeof(const void*)
<= sizeof(unsigned long)),
unsigned long, unsigned long long>::__type _UIntPtrType;
__s = _M_insert_int(__s, __io, __fill,
reinterpret_cast<_UIntPtrType>(__v));
__io.flags(__flags);
return __s;
}
template<typename _CharT, typename _Traits>
void
__pad<_CharT, _Traits>::_S_pad(ios_base& __io, _CharT __fill,
_CharT* __news, const _CharT* __olds,
streamsize __newlen, streamsize __oldlen)
{
const size_t __plen = static_cast<size_t>(__newlen - __oldlen);
const ios_base::fmtflags __adjust = __io.flags() & ios_base::adjustfield;
if (__adjust == ios_base::left)
{
_Traits::copy(__news, __olds, __oldlen);
_Traits::assign(__news + __oldlen, __plen, __fill);
return;
}
size_t __mod = 0;
if (__adjust == ios_base::internal)
{
const locale& __loc = __io._M_getloc();
const ctype<_CharT>& __ctype = use_facet<ctype<_CharT> >(__loc);
if (__ctype.widen('-') == __olds[0]
|| __ctype.widen('+') == __olds[0])
{
__news[0] = __olds[0];
__mod = 1;
++__news;
}
else if (__ctype.widen('0') == __olds[0]
&& __oldlen > 1
&& (__ctype.widen('x') == __olds[1]
|| __ctype.widen('X') == __olds[1]))
{
__news[0] = __olds[0];
__news[1] = __olds[1];
__mod = 2;
__news += 2;
}
}
_Traits::assign(__news, __plen, __fill);
_Traits::copy(__news + __plen, __olds + __mod, __oldlen - __mod);
}
template<typename _CharT>
_CharT*
__add_grouping(_CharT* __s, _CharT __sep,
const char* __gbeg, size_t __gsize,
const _CharT* __first, const _CharT* __last)
{
size_t __idx = 0;
size_t __ctr = 0;
while (__last - __first > __gbeg[__idx]
&& static_cast<signed char>(__gbeg[__idx]) > 0
&& __gbeg[__idx] != __gnu_cxx::__numeric_traits<char>::__max)
{
__last -= __gbeg[__idx];
__idx < __gsize - 1 ? ++__idx : ++__ctr;
}
while (__first != __last)
*__s++ = *__first++;
while (__ctr--)
{
*__s++ = __sep;
for (char __i = __gbeg[__idx]; __i > 0; --__i)
*__s++ = *__first++;
}
while (__idx--)
{
*__s++ = __sep;
for (char __i = __gbeg[__idx]; __i > 0; --__i)
*__s++ = *__first++;
}
return __s;
}
extern template class __cxx11:: numpunct<char>;
extern template class __cxx11:: numpunct_byname<char>;
extern template class num_get<char>;
extern template class num_put<char>;
extern template class ctype_byname<char>;
extern template
const ctype<char>*
__try_use_facet<ctype<char> >(const locale&) noexcept;
extern template
const numpunct<char>*
__try_use_facet<numpunct<char> >(const locale&) noexcept;
extern template
const num_put<char>*
__try_use_facet<num_put<char> >(const locale&) noexcept;
extern template
const num_get<char>*
__try_use_facet<num_get<char> >(const locale&) noexcept;
extern template
const ctype<char>&
use_facet<ctype<char> >(const locale&);
extern template
const numpunct<char>&
use_facet<numpunct<char> >(const locale&);
extern template
const num_put<char>&
use_facet<num_put<char> >(const locale&);
extern template
const num_get<char>&
use_facet<num_get<char> >(const locale&);
extern template
bool
has_facet<ctype<char> >(const locale&);
extern template
bool
has_facet<numpunct<char> >(const locale&);
extern template
bool
has_facet<num_put<char> >(const locale&);
extern template
bool
has_facet<num_get<char> >(const locale&);
extern template class __cxx11:: numpunct<wchar_t>;
extern template class __cxx11:: numpunct_byname<wchar_t>;
extern template class num_get<wchar_t>;
extern template class num_put<wchar_t>;
extern template class ctype_byname<wchar_t>;
extern template
const ctype<wchar_t>*
__try_use_facet<ctype<wchar_t> >(const locale&) noexcept;
extern template
const numpunct<wchar_t>*
__try_use_facet<numpunct<wchar_t> >(const locale&) noexcept;
extern template
const num_put<wchar_t>*
__try_use_facet<num_put<wchar_t> >(const locale&) noexcept;
extern template
const num_get<wchar_t>*
__try_use_facet<num_get<wchar_t> >(const locale&) noexcept;
extern template
const ctype<wchar_t>&
use_facet<ctype<wchar_t> >(const locale&);
extern template
const numpunct<wchar_t>&
use_facet<numpunct<wchar_t> >(const locale&);
extern template
const num_put<wchar_t>&
use_facet<num_put<wchar_t> >(const locale&);
extern template
const num_get<wchar_t>&
use_facet<num_get<wchar_t> >(const locale&);
extern template
bool
has_facet<ctype<wchar_t> >(const locale&);
extern template
bool
has_facet<numpunct<wchar_t> >(const locale&);
extern template
bool
has_facet<num_put<wchar_t> >(const locale&);
extern template
bool
has_facet<num_get<wchar_t> >(const locale&);
}
namespace std
{
using ::clock_t;
using ::time_t;
using ::tm;
using ::clock;
using ::difftime;
using ::mktime;
using ::time;
using ::asctime;
using ::ctime;
using ::gmtime;
using ::localtime;
using ::strftime;
}
namespace std
{
using ::timespec;
using ::timespec_get;
}
namespace std __attribute__ ((__visibility__ ("default")))
{
class time_base
{
public:
enum dateorder { no_order, dmy, mdy, ymd, ydm };
};
template<typename _CharT>
struct __timepunct_cache : public locale::facet
{
static const _CharT* _S_timezones[14];
const _CharT* _M_date_format;
const _CharT* _M_date_era_format;
const _CharT* _M_time_format;
const _CharT* _M_time_era_format;
const _CharT* _M_date_time_format;
const _CharT* _M_date_time_era_format;
const _CharT* _M_am;
const _CharT* _M_pm;
const _CharT* _M_am_pm_format;
const _CharT* _M_day1;
const _CharT* _M_day2;
const _CharT* _M_day3;
const _CharT* _M_day4;
const _CharT* _M_day5;
const _CharT* _M_day6;
const _CharT* _M_day7;
const _CharT* _M_aday1;
const _CharT* _M_aday2;
const _CharT* _M_aday3;
const _CharT* _M_aday4;
const _CharT* _M_aday5;
const _CharT* _M_aday6;
const _CharT* _M_aday7;
const _CharT* _M_month01;
const _CharT* _M_month02;
const _CharT* _M_month03;
const _CharT* _M_month04;
const _CharT* _M_month05;
const _CharT* _M_month06;
const _CharT* _M_month07;
const _CharT* _M_month08;
const _CharT* _M_month09;
const _CharT* _M_month10;
const _CharT* _M_month11;
const _CharT* _M_month12;
const _CharT* _M_amonth01;
const _CharT* _M_amonth02;
const _CharT* _M_amonth03;
const _CharT* _M_amonth04;
const _CharT* _M_amonth05;
const _CharT* _M_amonth06;
const _CharT* _M_amonth07;
const _CharT* _M_amonth08;
const _CharT* _M_amonth09;
const _CharT* _M_amonth10;
const _CharT* _M_amonth11;
const _CharT* _M_amonth12;
bool _M_allocated;
__timepunct_cache(size_t __refs = 0) : facet(__refs),
_M_date_format(0), _M_date_era_format(0), _M_time_format(0),
_M_time_era_format(0), _M_date_time_format(0),
_M_date_time_era_format(0), _M_am(0), _M_pm(0),
_M_am_pm_format(0), _M_day1(0), _M_day2(0), _M_day3(0),
_M_day4(0), _M_day5(0), _M_day6(0), _M_day7(0),
_M_aday1(0), _M_aday2(0), _M_aday3(0), _M_aday4(0),
_M_aday5(0), _M_aday6(0), _M_aday7(0), _M_month01(0),
_M_month02(0), _M_month03(0), _M_month04(0), _M_month05(0),
_M_month06(0), _M_month07(0), _M_month08(0), _M_month09(0),
_M_month10(0), _M_month11(0), _M_month12(0), _M_amonth01(0),
_M_amonth02(0), _M_amonth03(0), _M_amonth04(0),
_M_amonth05(0), _M_amonth06(0), _M_amonth07(0),
_M_amonth08(0), _M_amonth09(0), _M_amonth10(0),
_M_amonth11(0), _M_amonth12(0), _M_allocated(false)
{ }
~__timepunct_cache();
private:
__timepunct_cache&
operator=(const __timepunct_cache&);
explicit
__timepunct_cache(const __timepunct_cache&);
};
template<typename _CharT>
__timepunct_cache<_CharT>::~__timepunct_cache()
{
if (_M_allocated)
{
}
}
template<>
const char*
__timepunct_cache<char>::_S_timezones[14];
template<>
const wchar_t*
__timepunct_cache<wchar_t>::_S_timezones[14];
template<typename _CharT>
const _CharT* __timepunct_cache<_CharT>::_S_timezones[14];
template<typename _CharT>
class __timepunct : public locale::facet
{
public:
typedef _CharT __char_type;
typedef __timepunct_cache<_CharT> __cache_type;
protected:
__cache_type* _M_data;
__c_locale _M_c_locale_timepunct;
const char* _M_name_timepunct;
public:
static locale::id id;
explicit
__timepunct(size_t __refs = 0);
explicit
__timepunct(__cache_type* __cache, size_t __refs = 0);
explicit
__timepunct(__c_locale __cloc, const char* __s, size_t __refs = 0);
void
_M_put(_CharT* __s, size_t __maxlen, const _CharT* __format,
const tm* __tm) const throw ();
void
_M_date_formats(const _CharT** __date) const
{
__date[0] = _M_data->_M_date_format;
__date[1] = _M_data->_M_date_era_format;
}
void
_M_time_formats(const _CharT** __time) const
{
__time[0] = _M_data->_M_time_format;
__time[1] = _M_data->_M_time_era_format;
}
void
_M_date_time_formats(const _CharT** __dt) const
{
__dt[0] = _M_data->_M_date_time_format;
__dt[1] = _M_data->_M_date_time_era_format;
}
void
_M_am_pm_format(const _CharT*) const
{ }
void
_M_am_pm_format(const _CharT** __ampm_format) const
{
__ampm_format[0] = _M_data->_M_am_pm_format;
}
void
_M_am_pm(const _CharT** __ampm) const
{
__ampm[0] = _M_data->_M_am;
__ampm[1] = _M_data->_M_pm;
}
void
_M_days(const _CharT** __days) const
{
__days[0] = _M_data->_M_day1;
__days[1] = _M_data->_M_day2;
__days[2] = _M_data->_M_day3;
__days[3] = _M_data->_M_day4;
__days[4] = _M_data->_M_day5;
__days[5] = _M_data->_M_day6;
__days[6] = _M_data->_M_day7;
}
void
_M_days_abbreviated(const _CharT** __days) const
{
__days[0] = _M_data->_M_aday1;
__days[1] = _M_data->_M_aday2;
__days[2] = _M_data->_M_aday3;
__days[3] = _M_data->_M_aday4;
__days[4] = _M_data->_M_aday5;
__days[5] = _M_data->_M_aday6;
__days[6] = _M_data->_M_aday7;
}
void
_M_months(const _CharT** __months) const
{
__months[0] = _M_data->_M_month01;
__months[1] = _M_data->_M_month02;
__months[2] = _M_data->_M_month03;
__months[3] = _M_data->_M_month04;
__months[4] = _M_data->_M_month05;
__months[5] = _M_data->_M_month06;
__months[6] = _M_data->_M_month07;
__months[7] = _M_data->_M_month08;
__months[8] = _M_data->_M_month09;
__months[9] = _M_data->_M_month10;
__months[10] = _M_data->_M_month11;
__months[11] = _M_data->_M_month12;
}
void
_M_months_abbreviated(const _CharT** __months) const
{
__months[0] = _M_data->_M_amonth01;
__months[1] = _M_data->_M_amonth02;
__months[2] = _M_data->_M_amonth03;
__months[3] = _M_data->_M_amonth04;
__months[4] = _M_data->_M_amonth05;
__months[5] = _M_data->_M_amonth06;
__months[6] = _M_data->_M_amonth07;
__months[7] = _M_data->_M_amonth08;
__months[8] = _M_data->_M_amonth09;
__months[9] = _M_data->_M_amonth10;
__months[10] = _M_data->_M_amonth11;
__months[11] = _M_data->_M_amonth12;
}
protected:
virtual
~__timepunct();
void
_M_initialize_timepunct(__c_locale __cloc = 0);
};
template<typename _CharT>
locale::id __timepunct<_CharT>::id;
template<>
void
__timepunct<char>::_M_initialize_timepunct(__c_locale __cloc);
template<>
void
__timepunct<char>::_M_put(char*, size_t, const char*, const tm*) const throw ();
template<>
void
__timepunct<wchar_t>::_M_initialize_timepunct(__c_locale __cloc);
template<>
void
__timepunct<wchar_t>::_M_put(wchar_t*, size_t, const wchar_t*,
const tm*) const throw ();
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _CharT>
__timepunct<_CharT>::__timepunct(size_t __refs)
: facet(__refs), _M_data(0), _M_c_locale_timepunct(0),
_M_name_timepunct(_S_get_c_name())
{ _M_initialize_timepunct(); }
template<typename _CharT>
__timepunct<_CharT>::__timepunct(__cache_type* __cache, size_t __refs)
: facet(__refs), _M_data(__cache), _M_c_locale_timepunct(0),
_M_name_timepunct(_S_get_c_name())
{ _M_initialize_timepunct(); }
template<typename _CharT>
__timepunct<_CharT>::__timepunct(__c_locale __cloc, const char* __s,
size_t __refs)
: facet(__refs), _M_data(0), _M_c_locale_timepunct(0),
_M_name_timepunct(0)
{
if (__builtin_strcmp(__s, _S_get_c_name()) != 0)
{
const size_t __len = __builtin_strlen(__s) + 1;
char* __tmp = new char[__len];
__builtin_memcpy(__tmp, __s, __len);
_M_name_timepunct = __tmp;
}
else
_M_name_timepunct = _S_get_c_name();
try
{ _M_initialize_timepunct(__cloc); }
catch(...)
{
if (_M_name_timepunct != _S_get_c_name())
delete [] _M_name_timepunct;
throw;
}
}
template<typename _CharT>
__timepunct<_CharT>::~__timepunct()
{
if (_M_name_timepunct != _S_get_c_name())
delete [] _M_name_timepunct;
delete _M_data;
_S_destroy_c_locale(_M_c_locale_timepunct);
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
struct __time_get_state
{
void
_M_finalize_state(tm* __tm);
unsigned int _M_have_I : 1;
unsigned int _M_have_wday : 1;
unsigned int _M_have_yday : 1;
unsigned int _M_have_mon : 1;
unsigned int _M_have_mday : 1;
unsigned int _M_have_uweek : 1;
unsigned int _M_have_wweek : 1;
unsigned int _M_have_century : 1;
unsigned int _M_is_pm : 1;
unsigned int _M_want_century : 1;
unsigned int _M_want_xday : 1;
unsigned int _M_pad1 : 5;
unsigned int _M_week_no : 6;
unsigned int _M_pad2 : 10;
int _M_century;
int _M_pad3;
};
namespace __cxx11 {
template<typename _CharT, typename _InIter>
class time_get : public locale::facet, public time_base
{
public:
typedef _CharT char_type;
typedef _InIter iter_type;
static locale::id id;
explicit
time_get(size_t __refs = 0)
: facet (__refs) { }
dateorder
date_order() const
{ return this->do_date_order(); }
iter_type
get_time(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm) const
{ return this->do_get_time(__beg, __end, __io, __err, __tm); }
iter_type
get_date(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm) const
{ return this->do_get_date(__beg, __end, __io, __err, __tm); }
iter_type
get_weekday(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm) const
{ return this->do_get_weekday(__beg, __end, __io, __err, __tm); }
iter_type
get_monthname(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm) const
{ return this->do_get_monthname(__beg, __end, __io, __err, __tm); }
iter_type
get_year(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm) const
{ return this->do_get_year(__beg, __end, __io, __err, __tm); }
inline
iter_type get(iter_type __s, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm, char __format,
char __modifier = 0) const
{
return this->do_get(__s, __end, __io, __err, __tm, __format,
__modifier);
}
iter_type get(iter_type __s, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm, const char_type* __fmt,
const char_type* __fmtend) const;
protected:
virtual
~time_get() { }
virtual dateorder
do_date_order() const;
virtual iter_type
do_get_time(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm) const;
virtual iter_type
do_get_date(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm) const;
virtual iter_type
do_get_weekday(iter_type __beg, iter_type __end, ios_base&,
ios_base::iostate& __err, tm* __tm) const;
virtual iter_type
do_get_monthname(iter_type __beg, iter_type __end, ios_base&,
ios_base::iostate& __err, tm* __tm) const;
virtual iter_type
do_get_year(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm) const;
virtual
iter_type
do_get(iter_type __s, iter_type __end, ios_base& __f,
ios_base::iostate& __err, tm* __tm,
char __format, char __modifier) const;
iter_type
_M_extract_num(iter_type __beg, iter_type __end, int& __member,
int __min, int __max, size_t __len,
ios_base& __io, ios_base::iostate& __err) const;
iter_type
_M_extract_name(iter_type __beg, iter_type __end, int& __member,
const _CharT** __names, size_t __indexlen,
ios_base& __io, ios_base::iostate& __err) const;
iter_type
_M_extract_wday_or_month(iter_type __beg, iter_type __end, int& __member,
const _CharT** __names, size_t __indexlen,
ios_base& __io, ios_base::iostate& __err) const;
iter_type
_M_extract_via_format(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm,
const _CharT* __format) const;
iter_type
_M_extract_via_format(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm,
const _CharT* __format,
__time_get_state &__state) const;
};
template<typename _CharT, typename _InIter>
locale::id time_get<_CharT, _InIter>::id;
template<typename _CharT, typename _InIter>
class time_get_byname : public time_get<_CharT, _InIter>
{
public:
typedef _CharT char_type;
typedef _InIter iter_type;
explicit
time_get_byname(const char*, size_t __refs = 0)
: time_get<_CharT, _InIter>(__refs) { }
explicit
time_get_byname(const string& __s, size_t __refs = 0)
: time_get_byname(__s.c_str(), __refs) { }
protected:
virtual
~time_get_byname() { }
};
}
template<typename _CharT, typename _OutIter>
class time_put : public locale::facet
{
public:
typedef _CharT char_type;
typedef _OutIter iter_type;
static locale::id id;
explicit
time_put(size_t __refs = 0)
: facet(__refs) { }
iter_type
put(iter_type __s, ios_base& __io, char_type __fill, const tm* __tm,
const _CharT* __beg, const _CharT* __end) const;
iter_type
put(iter_type __s, ios_base& __io, char_type __fill,
const tm* __tm, char __format, char __mod = 0) const
{ return this->do_put(__s, __io, __fill, __tm, __format, __mod); }
protected:
virtual
~time_put()
{ }
virtual iter_type
do_put(iter_type __s, ios_base& __io, char_type __fill, const tm* __tm,
char __format, char __mod) const;
};
template<typename _CharT, typename _OutIter>
locale::id time_put<_CharT, _OutIter>::id;
template<typename _CharT, typename _OutIter>
class time_put_byname : public time_put<_CharT, _OutIter>
{
public:
typedef _CharT char_type;
typedef _OutIter iter_type;
explicit
time_put_byname(const char*, size_t __refs = 0)
: time_put<_CharT, _OutIter>(__refs)
{ }
explicit
time_put_byname(const string& __s, size_t __refs = 0)
: time_put_byname(__s.c_str(), __refs) { }
protected:
virtual
~time_put_byname() { }
};
class money_base
{
public:
enum part { none, space, symbol, sign, value };
struct pattern { char field[4]; };
static const pattern _S_default_pattern;
enum
{
_S_minus,
_S_zero,
_S_end = 11
};
static const char* _S_atoms;
__attribute__ ((__const__)) static pattern
_S_construct_pattern(char __precedes, char __space, char __posn) throw ();
};
template<typename _CharT, bool _Intl>
struct __moneypunct_cache : public locale::facet
{
const char* _M_grouping;
size_t _M_grouping_size;
bool _M_use_grouping;
_CharT _M_decimal_point;
_CharT _M_thousands_sep;
const _CharT* _M_curr_symbol;
size_t _M_curr_symbol_size;
const _CharT* _M_positive_sign;
size_t _M_positive_sign_size;
const _CharT* _M_negative_sign;
size_t _M_negative_sign_size;
int _M_frac_digits;
money_base::pattern _M_pos_format;
money_base::pattern _M_neg_format;
_CharT _M_atoms[money_base::_S_end];
bool _M_allocated;
__moneypunct_cache(size_t __refs = 0) : facet(__refs),
_M_grouping(0), _M_grouping_size(0), _M_use_grouping(false),
_M_decimal_point(_CharT()), _M_thousands_sep(_CharT()),
_M_curr_symbol(0), _M_curr_symbol_size(0),
_M_positive_sign(0), _M_positive_sign_size(0),
_M_negative_sign(0), _M_negative_sign_size(0),
_M_frac_digits(0),
_M_pos_format(money_base::pattern()),
_M_neg_format(money_base::pattern()), _M_allocated(false)
{ }
~__moneypunct_cache();
void
_M_cache(const locale& __loc);
private:
__moneypunct_cache&
operator=(const __moneypunct_cache&);
explicit
__moneypunct_cache(const __moneypunct_cache&);
};
template<typename _CharT, bool _Intl>
__moneypunct_cache<_CharT, _Intl>::~__moneypunct_cache()
{
if (_M_allocated)
{
delete [] _M_grouping;
delete [] _M_curr_symbol;
delete [] _M_positive_sign;
delete [] _M_negative_sign;
}
}
namespace __cxx11 {
template<typename _CharT, bool _Intl>
class moneypunct : public locale::facet, public money_base
{
public:
typedef _CharT char_type;
typedef basic_string<_CharT> string_type;
typedef __moneypunct_cache<_CharT, _Intl> __cache_type;
private:
__cache_type* _M_data;
public:
static const bool intl = _Intl;
static locale::id id;
explicit
moneypunct(size_t __refs = 0)
: facet(__refs), _M_data(0)
{ _M_initialize_moneypunct(); }
explicit
moneypunct(__cache_type* __cache, size_t __refs = 0)
: facet(__refs), _M_data(__cache)
{ _M_initialize_moneypunct(); }
explicit
moneypunct(__c_locale __cloc, const char* __s, size_t __refs = 0)
: facet(__refs), _M_data(0)
{ _M_initialize_moneypunct(__cloc, __s); }
char_type
decimal_point() const
{ return this->do_decimal_point(); }
char_type
thousands_sep() const
{ return this->do_thousands_sep(); }
string
grouping() const
{ return this->do_grouping(); }
string_type
curr_symbol() const
{ return this->do_curr_symbol(); }
string_type
positive_sign() const
{ return this->do_positive_sign(); }
string_type
negative_sign() const
{ return this->do_negative_sign(); }
int
frac_digits() const
{ return this->do_frac_digits(); }
pattern
pos_format() const
{ return this->do_pos_format(); }
pattern
neg_format() const
{ return this->do_neg_format(); }
protected:
virtual
~moneypunct();
virtual char_type
do_decimal_point() const
{ return _M_data->_M_decimal_point; }
virtual char_type
do_thousands_sep() const
{ return _M_data->_M_thousands_sep; }
virtual string
do_grouping() const
{ return _M_data->_M_grouping; }
virtual string_type
do_curr_symbol() const
{ return _M_data->_M_curr_symbol; }
virtual string_type
do_positive_sign() const
{ return _M_data->_M_positive_sign; }
virtual string_type
do_negative_sign() const
{ return _M_data->_M_negative_sign; }
virtual int
do_frac_digits() const
{ return _M_data->_M_frac_digits; }
virtual pattern
do_pos_format() const
{ return _M_data->_M_pos_format; }
virtual pattern
do_neg_format() const
{ return _M_data->_M_neg_format; }
void
_M_initialize_moneypunct(__c_locale __cloc = 0,
const char* __name = 0);
};
template<typename _CharT, bool _Intl>
locale::id moneypunct<_CharT, _Intl>::id;
template<typename _CharT, bool _Intl>
const bool moneypunct<_CharT, _Intl>::intl;
template<>
moneypunct<char, true>::~moneypunct();
template<>
moneypunct<char, false>::~moneypunct();
template<>
void
moneypunct<char, true>::_M_initialize_moneypunct(__c_locale, const char*);
template<>
void
moneypunct<char, false>::_M_initialize_moneypunct(__c_locale, const char*);
template<>
moneypunct<wchar_t, true>::~moneypunct();
template<>
moneypunct<wchar_t, false>::~moneypunct();
template<>
void
moneypunct<wchar_t, true>::_M_initialize_moneypunct(__c_locale,
const char*);
template<>
void
moneypunct<wchar_t, false>::_M_initialize_moneypunct(__c_locale,
const char*);
template<typename _CharT, bool _Intl>
class moneypunct_byname : public moneypunct<_CharT, _Intl>
{
public:
typedef _CharT char_type;
typedef basic_string<_CharT> string_type;
static const bool intl = _Intl;
explicit
moneypunct_byname(const char* __s, size_t __refs = 0)
: moneypunct<_CharT, _Intl>(__refs)
{
if (__builtin_strcmp(__s, "C") != 0
&& __builtin_strcmp(__s, "POSIX") != 0)
{
__c_locale __tmp;
this->_S_create_c_locale(__tmp, __s);
this->_M_initialize_moneypunct(__tmp);
this->_S_destroy_c_locale(__tmp);
}
}
explicit
moneypunct_byname(const string& __s, size_t __refs = 0)
: moneypunct_byname(__s.c_str(), __refs) { }
protected:
virtual
~moneypunct_byname() { }
};
template<typename _CharT, bool _Intl>
const bool moneypunct_byname<_CharT, _Intl>::intl;
}
namespace __cxx11 {
template<typename _CharT, typename _InIter>
class money_get : public locale::facet
{
public:
typedef _CharT char_type;
typedef _InIter iter_type;
typedef basic_string<_CharT> string_type;
static locale::id id;
explicit
money_get(size_t __refs = 0) : facet(__refs) { }
iter_type
get(iter_type __s, iter_type __end, bool __intl, ios_base& __io,
ios_base::iostate& __err, long double& __units) const
{ return this->do_get(__s, __end, __intl, __io, __err, __units); }
iter_type
get(iter_type __s, iter_type __end, bool __intl, ios_base& __io,
ios_base::iostate& __err, string_type& __digits) const
{ return this->do_get(__s, __end, __intl, __io, __err, __digits); }
protected:
virtual
~money_get() { }
virtual iter_type
do_get(iter_type __s, iter_type __end, bool __intl, ios_base& __io,
ios_base::iostate& __err, long double& __units) const;
virtual iter_type
do_get(iter_type __s, iter_type __end, bool __intl, ios_base& __io,
ios_base::iostate& __err, string_type& __digits) const;
template<bool _Intl>
iter_type
_M_extract(iter_type __s, iter_type __end, ios_base& __io,
ios_base::iostate& __err, string& __digits) const;
};
template<typename _CharT, typename _InIter>
locale::id money_get<_CharT, _InIter>::id;
template<typename _CharT, typename _OutIter>
class money_put : public locale::facet
{
public:
typedef _CharT char_type;
typedef _OutIter iter_type;
typedef basic_string<_CharT> string_type;
static locale::id id;
explicit
money_put(size_t __refs = 0) : facet(__refs) { }
iter_type
put(iter_type __s, bool __intl, ios_base& __io,
char_type __fill, long double __units) const
{ return this->do_put(__s, __intl, __io, __fill, __units); }
iter_type
put(iter_type __s, bool __intl, ios_base& __io,
char_type __fill, const string_type& __digits) const
{ return this->do_put(__s, __intl, __io, __fill, __digits); }
protected:
virtual
~money_put() { }
virtual iter_type
do_put(iter_type __s, bool __intl, ios_base& __io, char_type __fill,
long double __units) const;
virtual iter_type
do_put(iter_type __s, bool __intl, ios_base& __io, char_type __fill,
const string_type& __digits) const;
template<bool _Intl>
iter_type
_M_insert(iter_type __s, ios_base& __io, char_type __fill,
const string_type& __digits) const;
};
template<typename _CharT, typename _OutIter>
locale::id money_put<_CharT, _OutIter>::id;
}
struct messages_base
{
typedef int catalog;
};
namespace __cxx11 {
template<typename _CharT>
class messages : public locale::facet, public messages_base
{
public:
typedef _CharT char_type;
typedef basic_string<_CharT> string_type;
protected:
__c_locale _M_c_locale_messages;
const char* _M_name_messages;
public:
static locale::id id;
explicit
messages(size_t __refs = 0);
explicit
messages(__c_locale __cloc, const char* __s, size_t __refs = 0);
catalog
open(const basic_string<char>& __s, const locale& __loc) const
{ return this->do_open(__s, __loc); }
catalog
open(const basic_string<char>&, const locale&, const char*) const;
string_type
get(catalog __c, int __set, int __msgid, const string_type& __s) const
{ return this->do_get(__c, __set, __msgid, __s); }
void
close(catalog __c) const
{ return this->do_close(__c); }
protected:
virtual
~messages();
virtual catalog
do_open(const basic_string<char>&, const locale&) const;
virtual string_type
do_get(catalog, int, int, const string_type& __dfault) const;
virtual void
do_close(catalog) const;
char*
_M_convert_to_char(const string_type& __msg) const
{
return reinterpret_cast<char*>(const_cast<_CharT*>(__msg.c_str()));
}
string_type
_M_convert_from_char(char*) const
{
return string_type();
}
};
template<typename _CharT>
locale::id messages<_CharT>::id;
template<>
string
messages<char>::do_get(catalog, int, int, const string&) const;
template<>
wstring
messages<wchar_t>::do_get(catalog, int, int, const wstring&) const;
template<typename _CharT>
class messages_byname : public messages<_CharT>
{
public:
typedef _CharT char_type;
typedef basic_string<_CharT> string_type;
explicit
messages_byname(const char* __s, size_t __refs = 0);
explicit
messages_byname(const string& __s, size_t __refs = 0)
: messages_byname(__s.c_str(), __refs) { }
protected:
virtual
~messages_byname()
{ }
};
}
}
extern "C" {
extern char *gettext (const char *__msgid)
noexcept (true) __attribute__ ((__format_arg__ (1)));
extern char *dgettext (const char *__domainname, const char *__msgid)
noexcept (true) __attribute__ ((__format_arg__ (2)));
extern char *__dgettext (const char *__domainname, const char *__msgid)
noexcept (true) __attribute__ ((__format_arg__ (2)));
extern char *dcgettext (const char *__domainname,
const char *__msgid, int __category)
noexcept (true) __attribute__ ((__format_arg__ (2)));
extern char *__dcgettext (const char *__domainname,
const char *__msgid, int __category)
noexcept (true) __attribute__ ((__format_arg__ (2)));
extern char *ngettext (const char *__msgid1, const char *__msgid2,
unsigned long int __n)
noexcept (true) __attribute__ ((__format_arg__ (1))) __attribute__ ((__format_arg__ (2)));
extern char *dngettext (const char *__domainname, const char *__msgid1,
const char *__msgid2, unsigned long int __n)
noexcept (true) __attribute__ ((__format_arg__ (2))) __attribute__ ((__format_arg__ (3)));
extern char *dcngettext (const char *__domainname, const char *__msgid1,
const char *__msgid2, unsigned long int __n,
int __category)
noexcept (true) __attribute__ ((__format_arg__ (2))) __attribute__ ((__format_arg__ (3)));
extern char *textdomain (const char *__domainname) noexcept (true);
extern char *bindtextdomain (const char *__domainname,
const char *__dirname) noexcept (true);
extern char *bind_textdomain_codeset (const char *__domainname,
const char *__codeset) noexcept (true);
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _CharT>
messages<_CharT>::messages(size_t __refs)
: facet(__refs), _M_c_locale_messages(_S_get_c_locale()),
_M_name_messages(_S_get_c_name())
{ }
template<typename _CharT>
messages<_CharT>::messages(__c_locale __cloc, const char* __s,
size_t __refs)
: facet(__refs), _M_c_locale_messages(0), _M_name_messages(0)
{
if (__builtin_strcmp(__s, _S_get_c_name()) != 0)
{
const size_t __len = __builtin_strlen(__s) + 1;
char* __tmp = new char[__len];
__builtin_memcpy(__tmp, __s, __len);
_M_name_messages = __tmp;
}
else
_M_name_messages = _S_get_c_name();
_M_c_locale_messages = _S_clone_c_locale(__cloc);
}
template<typename _CharT>
typename messages<_CharT>::catalog
messages<_CharT>::open(const basic_string<char>& __s, const locale& __loc,
const char* __dir) const
{
bindtextdomain(__s.c_str(), __dir);
return this->do_open(__s, __loc);
}
template<typename _CharT>
messages<_CharT>::~messages()
{
if (_M_name_messages != _S_get_c_name())
delete [] _M_name_messages;
_S_destroy_c_locale(_M_c_locale_messages);
}
template<typename _CharT>
typename messages<_CharT>::catalog
messages<_CharT>::do_open(const basic_string<char>& __s,
const locale&) const
{
textdomain(__s.c_str());
return 0;
}
template<typename _CharT>
void
messages<_CharT>::do_close(catalog) const
{ }
template<typename _CharT>
messages_byname<_CharT>::messages_byname(const char* __s, size_t __refs)
: messages<_CharT>(__refs)
{
if (this->_M_name_messages != locale::facet::_S_get_c_name())
{
delete [] this->_M_name_messages;
if (__builtin_strcmp(__s, locale::facet::_S_get_c_name()) != 0)
{
const size_t __len = __builtin_strlen(__s) + 1;
char* __tmp = new char[__len];
__builtin_memcpy(__tmp, __s, __len);
this->_M_name_messages = __tmp;
}
else
this->_M_name_messages = locale::facet::_S_get_c_name();
}
if (__builtin_strcmp(__s, "C") != 0
&& __builtin_strcmp(__s, "POSIX") != 0)
{
this->_S_destroy_c_locale(this->_M_c_locale_messages);
this->_S_create_c_locale(this->_M_c_locale_messages, __s);
}
}
template<>
typename messages<char>::catalog
messages<char>::do_open(const basic_string<char>&,
const locale&) const;
template<>
void
messages<char>::do_close(catalog) const;
template<>
typename messages<wchar_t>::catalog
messages<wchar_t>::do_open(const basic_string<char>&,
const locale&) const;
template<>
void
messages<wchar_t>::do_close(catalog) const;
}
namespace std __attribute__ ((__visibility__ ("default")))
{
class codecvt_base
{
public:
enum result
{
ok,
partial,
error,
noconv
};
};
template<typename _InternT, typename _ExternT, typename _StateT>
class __codecvt_abstract_base
: public locale::facet, public codecvt_base
{
public:
typedef codecvt_base::result result;
typedef _InternT intern_type;
typedef _ExternT extern_type;
typedef _StateT state_type;
result
out(state_type& __state, const intern_type* __from,
const intern_type* __from_end, const intern_type*& __from_next,
extern_type* __to, extern_type* __to_end,
extern_type*& __to_next) const
{
return this->do_out(__state, __from, __from_end, __from_next,
__to, __to_end, __to_next);
}
result
unshift(state_type& __state, extern_type* __to, extern_type* __to_end,
extern_type*& __to_next) const
{ return this->do_unshift(__state, __to,__to_end,__to_next); }
result
in(state_type& __state, const extern_type* __from,
const extern_type* __from_end, const extern_type*& __from_next,
intern_type* __to, intern_type* __to_end,
intern_type*& __to_next) const
{
return this->do_in(__state, __from, __from_end, __from_next,
__to, __to_end, __to_next);
}
int
encoding() const throw()
{ return this->do_encoding(); }
bool
always_noconv() const throw()
{ return this->do_always_noconv(); }
int
length(state_type& __state, const extern_type* __from,
const extern_type* __end, size_t __max) const
{ return this->do_length(__state, __from, __end, __max); }
int
max_length() const throw()
{ return this->do_max_length(); }
protected:
explicit
__codecvt_abstract_base(size_t __refs = 0) : locale::facet(__refs) { }
virtual
~__codecvt_abstract_base() { }
virtual result
do_out(state_type& __state, const intern_type* __from,
const intern_type* __from_end, const intern_type*& __from_next,
extern_type* __to, extern_type* __to_end,
extern_type*& __to_next) const = 0;
virtual result
do_unshift(state_type& __state, extern_type* __to,
extern_type* __to_end, extern_type*& __to_next) const = 0;
virtual result
do_in(state_type& __state, const extern_type* __from,
const extern_type* __from_end, const extern_type*& __from_next,
intern_type* __to, intern_type* __to_end,
intern_type*& __to_next) const = 0;
virtual int
do_encoding() const throw() = 0;
virtual bool
do_always_noconv() const throw() = 0;
virtual int
do_length(state_type&, const extern_type* __from,
const extern_type* __end, size_t __max) const = 0;
virtual int
do_max_length() const throw() = 0;
};
template<typename _InternT, typename _ExternT, typename _StateT>
class codecvt
: public __codecvt_abstract_base<_InternT, _ExternT, _StateT>
{
public:
typedef codecvt_base::result result;
typedef _InternT intern_type;
typedef _ExternT extern_type;
typedef _StateT state_type;
protected:
__c_locale _M_c_locale_codecvt;
public:
static locale::id id;
explicit
codecvt(size_t __refs = 0)
: __codecvt_abstract_base<_InternT, _ExternT, _StateT> (__refs),
_M_c_locale_codecvt(0)
{ }
explicit
codecvt(__c_locale __cloc, size_t __refs = 0);
protected:
virtual
~codecvt() { }
virtual result
do_out(state_type& __state, const intern_type* __from,
const intern_type* __from_end, const intern_type*& __from_next,
extern_type* __to, extern_type* __to_end,
extern_type*& __to_next) const;
virtual result
do_unshift(state_type& __state, extern_type* __to,
extern_type* __to_end, extern_type*& __to_next) const;
virtual result
do_in(state_type& __state, const extern_type* __from,
const extern_type* __from_end, const extern_type*& __from_next,
intern_type* __to, intern_type* __to_end,
intern_type*& __to_next) const;
virtual int
do_encoding() const throw();
virtual bool
do_always_noconv() const throw();
virtual int
do_length(state_type&, const extern_type* __from,
const extern_type* __end, size_t __max) const;
virtual int
do_max_length() const throw();
};
template<typename _InternT, typename _ExternT, typename _StateT>
locale::id codecvt<_InternT, _ExternT, _StateT>::id;
template<>
class codecvt<char, char, mbstate_t>
: public __codecvt_abstract_base<char, char, mbstate_t>
{
friend class messages<char>;
public:
typedef char intern_type;
typedef char extern_type;
typedef mbstate_t state_type;
protected:
__c_locale _M_c_locale_codecvt;
public:
static locale::id id;
explicit
codecvt(size_t __refs = 0);
explicit
codecvt(__c_locale __cloc, size_t __refs = 0);
protected:
virtual
~codecvt();
virtual result
do_out(state_type& __state, const intern_type* __from,
const intern_type* __from_end, const intern_type*& __from_next,
extern_type* __to, extern_type* __to_end,
extern_type*& __to_next) const;
virtual result
do_unshift(state_type& __state, extern_type* __to,
extern_type* __to_end, extern_type*& __to_next) const;
virtual result
do_in(state_type& __state, const extern_type* __from,
const extern_type* __from_end, const extern_type*& __from_next,
intern_type* __to, intern_type* __to_end,
intern_type*& __to_next) const;
virtual int
do_encoding() const throw();
virtual bool
do_always_noconv() const throw();
virtual int
do_length(state_type&, const extern_type* __from,
const extern_type* __end, size_t __max) const;
virtual int
do_max_length() const throw();
};
template<>
class codecvt<wchar_t, char, mbstate_t>
: public __codecvt_abstract_base<wchar_t, char, mbstate_t>
{
friend class messages<wchar_t>;
public:
typedef wchar_t intern_type;
typedef char extern_type;
typedef mbstate_t state_type;
protected:
__c_locale _M_c_locale_codecvt;
public:
static locale::id id;
explicit
codecvt(size_t __refs = 0);
explicit
codecvt(__c_locale __cloc, size_t __refs = 0);
protected:
virtual
~codecvt();
virtual result
do_out(state_type& __state, const intern_type* __from,
const intern_type* __from_end, const intern_type*& __from_next,
extern_type* __to, extern_type* __to_end,
extern_type*& __to_next) const;
virtual result
do_unshift(state_type& __state,
extern_type* __to, extern_type* __to_end,
extern_type*& __to_next) const;
virtual result
do_in(state_type& __state,
const extern_type* __from, const extern_type* __from_end,
const extern_type*& __from_next,
intern_type* __to, intern_type* __to_end,
intern_type*& __to_next) const;
virtual
int do_encoding() const throw();
virtual
bool do_always_noconv() const throw();
virtual
int do_length(state_type&, const extern_type* __from,
const extern_type* __end, size_t __max) const;
virtual int
do_max_length() const throw();
};
template<>
class codecvt<char16_t, char, mbstate_t>
: public __codecvt_abstract_base<char16_t, char, mbstate_t>
{
public:
typedef char16_t intern_type;
typedef char extern_type;
typedef mbstate_t state_type;
public:
static locale::id id;
explicit
codecvt(size_t __refs = 0)
: __codecvt_abstract_base<char16_t, char, mbstate_t>(__refs) { }
protected:
virtual
~codecvt();
virtual result
do_out(state_type& __state, const intern_type* __from,
const intern_type* __from_end, const intern_type*& __from_next,
extern_type* __to, extern_type* __to_end,
extern_type*& __to_next) const;
virtual result
do_unshift(state_type& __state,
extern_type* __to, extern_type* __to_end,
extern_type*& __to_next) const;
virtual result
do_in(state_type& __state,
const extern_type* __from, const extern_type* __from_end,
const extern_type*& __from_next,
intern_type* __to, intern_type* __to_end,
intern_type*& __to_next) const;
virtual
int do_encoding() const throw();
virtual
bool do_always_noconv() const throw();
virtual
int do_length(state_type&, const extern_type* __from,
const extern_type* __end, size_t __max) const;
virtual int
do_max_length() const throw();
};
template<>
class codecvt<char32_t, char, mbstate_t>
: public __codecvt_abstract_base<char32_t, char, mbstate_t>
{
public:
typedef char32_t intern_type;
typedef char extern_type;
typedef mbstate_t state_type;
public:
static locale::id id;
explicit
codecvt(size_t __refs = 0)
: __codecvt_abstract_base<char32_t, char, mbstate_t>(__refs) { }
protected:
virtual
~codecvt();
virtual result
do_out(state_type& __state, const intern_type* __from,
const intern_type* __from_end, const intern_type*& __from_next,
extern_type* __to, extern_type* __to_end,
extern_type*& __to_next) const;
virtual result
do_unshift(state_type& __state,
extern_type* __to, extern_type* __to_end,
extern_type*& __to_next) const;
virtual result
do_in(state_type& __state,
const extern_type* __from, const extern_type* __from_end,
const extern_type*& __from_next,
intern_type* __to, intern_type* __to_end,
intern_type*& __to_next) const;
virtual
int do_encoding() const throw();
virtual
bool do_always_noconv() const throw();
virtual
int do_length(state_type&, const extern_type* __from,
const extern_type* __end, size_t __max) const;
virtual int
do_max_length() const throw();
};
template<typename _InternT, typename _ExternT, typename _StateT>
class codecvt_byname : public codecvt<_InternT, _ExternT, _StateT>
{
public:
explicit
codecvt_byname(const char* __s, size_t __refs = 0)
: codecvt<_InternT, _ExternT, _StateT>(__refs)
{
if (__builtin_strcmp(__s, "C") != 0
&& __builtin_strcmp(__s, "POSIX") != 0)
{
this->_S_destroy_c_locale(this->_M_c_locale_codecvt);
this->_S_create_c_locale(this->_M_c_locale_codecvt, __s);
}
}
explicit
codecvt_byname(const string& __s, size_t __refs = 0)
: codecvt_byname(__s.c_str(), __refs) { }
protected:
virtual
~codecvt_byname() { }
};
template<>
class codecvt_byname<char16_t, char, mbstate_t>
: public codecvt<char16_t, char, mbstate_t>
{
public:
explicit
codecvt_byname(const char*, size_t __refs = 0)
: codecvt<char16_t, char, mbstate_t>(__refs) { }
explicit
codecvt_byname(const string& __s, size_t __refs = 0)
: codecvt_byname(__s.c_str(), __refs) { }
protected:
virtual
~codecvt_byname() { }
};
template<>
class codecvt_byname<char32_t, char, mbstate_t>
: public codecvt<char32_t, char, mbstate_t>
{
public:
explicit
codecvt_byname(const char*, size_t __refs = 0)
: codecvt<char32_t, char, mbstate_t>(__refs) { }
explicit
codecvt_byname(const string& __s, size_t __refs = 0)
: codecvt_byname(__s.c_str(), __refs) { }
protected:
virtual
~codecvt_byname() { }
};
extern template class codecvt_byname<char, char, mbstate_t>;
extern template
const codecvt<char, char, mbstate_t>&
use_facet<codecvt<char, char, mbstate_t> >(const locale&);
extern template
bool
has_facet<codecvt<char, char, mbstate_t> >(const locale&);
extern template class codecvt_byname<wchar_t, char, mbstate_t>;
extern template
const codecvt<wchar_t, char, mbstate_t>&
use_facet<codecvt<wchar_t, char, mbstate_t> >(const locale&);
extern template
bool
has_facet<codecvt<wchar_t, char, mbstate_t> >(const locale&);
extern template class codecvt_byname<char16_t, char, mbstate_t>;
extern template class codecvt_byname<char32_t, char, mbstate_t>;
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _CharT, bool _Intl>
struct __use_cache<__moneypunct_cache<_CharT, _Intl> >
{
const __moneypunct_cache<_CharT, _Intl>*
operator() (const locale& __loc) const
{
const size_t __i = moneypunct<_CharT, _Intl>::id._M_id();
const locale::facet** __caches = __loc._M_impl->_M_caches;
if (!__caches[__i])
{
__moneypunct_cache<_CharT, _Intl>* __tmp = 0;
try
{
__tmp = new __moneypunct_cache<_CharT, _Intl>;
__tmp->_M_cache(__loc);
}
catch(...)
{
delete __tmp;
throw;
}
__loc._M_impl->_M_install_cache(__tmp, __i);
}
return static_cast<
const __moneypunct_cache<_CharT, _Intl>*>(__caches[__i]);
}
};
template<typename _CharT, bool _Intl>
void
__moneypunct_cache<_CharT, _Intl>::_M_cache(const locale& __loc)
{
const moneypunct<_CharT, _Intl>& __mp =
use_facet<moneypunct<_CharT, _Intl> >(__loc);
struct _Scoped_str
{
size_t _M_len;
_CharT* _M_str;
explicit
_Scoped_str(const basic_string<_CharT>& __str)
: _M_len(__str.size()), _M_str(new _CharT[_M_len])
{ __str.copy(_M_str, _M_len); }
~_Scoped_str() { delete[] _M_str; }
void
_M_release(const _CharT*& __p, size_t& __n)
{
__p = _M_str;
__n = _M_len;
_M_str = 0;
}
};
_Scoped_str __curr_symbol(__mp.curr_symbol());
_Scoped_str __positive_sign(__mp.positive_sign());
_Scoped_str __negative_sign(__mp.negative_sign());
const string& __g = __mp.grouping();
const size_t __g_size = __g.size();
char* const __grouping = new char[__g_size];
__g.copy(__grouping, __g_size);
_M_grouping = __grouping;
_M_grouping_size = __g_size;
_M_use_grouping = (__g_size
&& static_cast<signed char>(__grouping[0]) > 0
&& (__grouping[0]
!= __gnu_cxx::__numeric_traits<char>::__max));
_M_decimal_point = __mp.decimal_point();
_M_thousands_sep = __mp.thousands_sep();
__curr_symbol._M_release(_M_curr_symbol, _M_curr_symbol_size);
__positive_sign._M_release(_M_positive_sign, _M_positive_sign_size);
__negative_sign._M_release(_M_negative_sign, _M_negative_sign_size);
_M_frac_digits = __mp.frac_digits();
_M_pos_format = __mp.pos_format();
_M_neg_format = __mp.neg_format();
const ctype<_CharT>& __ct = use_facet<ctype<_CharT> >(__loc);
__ct.widen(money_base::_S_atoms,
money_base::_S_atoms + money_base::_S_end, _M_atoms);
_M_allocated = true;
}
namespace __cxx11 {
template<typename _CharT, typename _InIter>
template<bool _Intl>
_InIter
money_get<_CharT, _InIter>::
_M_extract(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, string& __units) const
{
typedef char_traits<_CharT> __traits_type;
typedef typename string_type::size_type size_type;
typedef money_base::part part;
typedef __moneypunct_cache<_CharT, _Intl> __cache_type;
const locale& __loc = __io._M_getloc();
const ctype<_CharT>& __ctype = use_facet<ctype<_CharT> >(__loc);
__use_cache<__cache_type> __uc;
const __cache_type* __lc = __uc(__loc);
const char_type* __lit = __lc->_M_atoms;
bool __negative = false;
size_type __sign_size = 0;
const bool __mandatory_sign = (__lc->_M_positive_sign_size
&& __lc->_M_negative_sign_size);
string __grouping_tmp;
if (__lc->_M_use_grouping)
__grouping_tmp.reserve(32);
int __last_pos = 0;
int __n = 0;
bool __testvalid = true;
bool __testdecfound = false;
string __res;
__res.reserve(32);
const char_type* __lit_zero = __lit + money_base::_S_zero;
const money_base::pattern __p = __lc->_M_neg_format;
for (int __i = 0; __i < 4 && __testvalid; ++__i)
{
const part __which = static_cast<part>(__p.field[__i]);
switch (__which)
{
case money_base::symbol:
if (__io.flags() & ios_base::showbase || __sign_size > 1
|| __i == 0
|| (__i == 1 && (__mandatory_sign
|| (static_cast<part>(__p.field[0])
== money_base::sign)
|| (static_cast<part>(__p.field[2])
== money_base::space)))
|| (__i == 2 && ((static_cast<part>(__p.field[3])
== money_base::value)
|| (__mandatory_sign
&& (static_cast<part>(__p.field[3])
== money_base::sign)))))
{
const size_type __len = __lc->_M_curr_symbol_size;
size_type __j = 0;
for (; __beg != __end && __j < __len
&& *__beg == __lc->_M_curr_symbol[__j];
++__beg, (void)++__j);
if (__j != __len
&& (__j || __io.flags() & ios_base::showbase))
__testvalid = false;
}
break;
case money_base::sign:
if (__lc->_M_positive_sign_size && __beg != __end
&& *__beg == __lc->_M_positive_sign[0])
{
__sign_size = __lc->_M_positive_sign_size;
++__beg;
}
else if (__lc->_M_negative_sign_size && __beg != __end
&& *__beg == __lc->_M_negative_sign[0])
{
__negative = true;
__sign_size = __lc->_M_negative_sign_size;
++__beg;
}
else if (__lc->_M_positive_sign_size
&& !__lc->_M_negative_sign_size)
__negative = true;
else if (__mandatory_sign)
__testvalid = false;
break;
case money_base::value:
for (; __beg != __end; ++__beg)
{
const char_type __c = *__beg;
const char_type* __q = __traits_type::find(__lit_zero,
10, __c);
if (__q != 0)
{
__res += money_base::_S_atoms[__q - __lit];
++__n;
}
else if (__c == __lc->_M_decimal_point
&& !__testdecfound)
{
if (__lc->_M_frac_digits <= 0)
break;
__last_pos = __n;
__n = 0;
__testdecfound = true;
}
else if (__lc->_M_use_grouping
&& __c == __lc->_M_thousands_sep
&& !__testdecfound)
{
if (__n)
{
__grouping_tmp += static_cast<char>(__n);
__n = 0;
}
else
{
__testvalid = false;
break;
}
}
else
break;
}
if (__res.empty())
__testvalid = false;
break;
case money_base::space:
if (__beg != __end && __ctype.is(ctype_base::space, *__beg))
++__beg;
else
__testvalid = false;
case money_base::none:
if (__i != 3)
for (; __beg != __end
&& __ctype.is(ctype_base::space, *__beg); ++__beg);
break;
}
}
if (__sign_size > 1 && __testvalid)
{
const char_type* __sign = __negative ? __lc->_M_negative_sign
: __lc->_M_positive_sign;
size_type __i = 1;
for (; __beg != __end && __i < __sign_size
&& *__beg == __sign[__i]; ++__beg, (void)++__i);
if (__i != __sign_size)
__testvalid = false;
}
if (__testvalid)
{
if (__res.size() > 1)
{
const size_type __first = __res.find_first_not_of('0');
const bool __only_zeros = __first == string::npos;
if (__first)
__res.erase(0, __only_zeros ? __res.size() - 1 : __first);
}
if (__negative && __res[0] != '0')
__res.insert(__res.begin(), '-');
if (__grouping_tmp.size())
{
__grouping_tmp += static_cast<char>(__testdecfound ? __last_pos
: __n);
if (!std::__verify_grouping(__lc->_M_grouping,
__lc->_M_grouping_size,
__grouping_tmp))
__err |= ios_base::failbit;
}
if (__testdecfound && __n != __lc->_M_frac_digits)
__testvalid = false;
}
if (!__testvalid)
__err |= ios_base::failbit;
else
__units.swap(__res);
if (__beg == __end)
__err |= ios_base::eofbit;
return __beg;
}
template<typename _CharT, typename _InIter>
_InIter
money_get<_CharT, _InIter>::
do_get(iter_type __beg, iter_type __end, bool __intl, ios_base& __io,
ios_base::iostate& __err, long double& __units) const
{
string __str;
__beg = __intl ? _M_extract<true>(__beg, __end, __io, __err, __str)
: _M_extract<false>(__beg, __end, __io, __err, __str);
std::__convert_to_v(__str.c_str(), __units, __err, _S_get_c_locale());
return __beg;
}
template<typename _CharT, typename _InIter>
_InIter
money_get<_CharT, _InIter>::
do_get(iter_type __beg, iter_type __end, bool __intl, ios_base& __io,
ios_base::iostate& __err, string_type& __digits) const
{
typedef typename string::size_type size_type;
const locale& __loc = __io._M_getloc();
const ctype<_CharT>& __ctype = use_facet<ctype<_CharT> >(__loc);
string __str;
__beg = __intl ? _M_extract<true>(__beg, __end, __io, __err, __str)
: _M_extract<false>(__beg, __end, __io, __err, __str);
const size_type __len = __str.size();
if (__len)
{
__digits.resize(__len);
__ctype.widen(__str.data(), __str.data() + __len, &__digits[0]);
}
return __beg;
}
template<typename _CharT, typename _OutIter>
template<bool _Intl>
_OutIter
money_put<_CharT, _OutIter>::
_M_insert(iter_type __s, ios_base& __io, char_type __fill,
const string_type& __digits) const
{
typedef typename string_type::size_type size_type;
typedef money_base::part part;
typedef __moneypunct_cache<_CharT, _Intl> __cache_type;
const locale& __loc = __io._M_getloc();
const ctype<_CharT>& __ctype = use_facet<ctype<_CharT> >(__loc);
__use_cache<__cache_type> __uc;
const __cache_type* __lc = __uc(__loc);
const char_type* __lit = __lc->_M_atoms;
const char_type* __beg = __digits.data();
money_base::pattern __p;
const char_type* __sign;
size_type __sign_size;
if (!(*__beg == __lit[money_base::_S_minus]))
{
__p = __lc->_M_pos_format;
__sign = __lc->_M_positive_sign;
__sign_size = __lc->_M_positive_sign_size;
}
else
{
__p = __lc->_M_neg_format;
__sign = __lc->_M_negative_sign;
__sign_size = __lc->_M_negative_sign_size;
if (__digits.size())
++__beg;
}
size_type __len = __ctype.scan_not(ctype_base::digit, __beg,
__beg + __digits.size()) - __beg;
if (__len)
{
string_type __value;
__value.reserve(2 * __len);
long __paddec = __len - __lc->_M_frac_digits;
if (__paddec > 0)
{
if (__lc->_M_frac_digits < 0)
__paddec = __len;
if (__lc->_M_grouping_size)
{
__value.assign(2 * __paddec, char_type());
_CharT* __vend =
std::__add_grouping(&__value[0], __lc->_M_thousands_sep,
__lc->_M_grouping,
__lc->_M_grouping_size,
__beg, __beg + __paddec);
__value.erase(__vend - &__value[0]);
}
else
__value.assign(__beg, __paddec);
}
if (__lc->_M_frac_digits > 0)
{
__value += __lc->_M_decimal_point;
if (__paddec >= 0)
__value.append(__beg + __paddec, __lc->_M_frac_digits);
else
{
__value.append(-__paddec, __lit[money_base::_S_zero]);
__value.append(__beg, __len);
}
}
const ios_base::fmtflags __f = __io.flags()
& ios_base::adjustfield;
__len = __value.size() + __sign_size;
__len += ((__io.flags() & ios_base::showbase)
? __lc->_M_curr_symbol_size : 0);
string_type __res;
__res.reserve(2 * __len);
const size_type __width = static_cast<size_type>(__io.width());
const bool __testipad = (__f == ios_base::internal
&& __len < __width);
for (int __i = 0; __i < 4; ++__i)
{
const part __which = static_cast<part>(__p.field[__i]);
switch (__which)
{
case money_base::symbol:
if (__io.flags() & ios_base::showbase)
__res.append(__lc->_M_curr_symbol,
__lc->_M_curr_symbol_size);
break;
case money_base::sign:
if (__sign_size)
__res += __sign[0];
break;
case money_base::value:
__res += __value;
break;
case money_base::space:
if (__testipad)
__res.append(__width - __len, __fill);
else
__res += __fill;
break;
case money_base::none:
if (__testipad)
__res.append(__width - __len, __fill);
break;
}
}
if (__sign_size > 1)
__res.append(__sign + 1, __sign_size - 1);
__len = __res.size();
if (__width > __len)
{
if (__f == ios_base::left)
__res.append(__width - __len, __fill);
else
__res.insert(0, __width - __len, __fill);
__len = __width;
}
__s = std::__write(__s, __res.data(), __len);
}
__io.width(0);
return __s;
}
template<typename _CharT, typename _OutIter>
_OutIter
money_put<_CharT, _OutIter>::
do_put(iter_type __s, bool __intl, ios_base& __io, char_type __fill,
long double __units) const
{
const locale __loc = __io.getloc();
const ctype<_CharT>& __ctype = use_facet<ctype<_CharT> >(__loc);
int __cs_size = 64;
char* __cs = static_cast<char*>(__builtin_alloca(__cs_size));
int __len = std::__convert_from_v(_S_get_c_locale(), __cs, __cs_size,
"%.*Lf", 0, __units);
if (__len >= __cs_size)
{
__cs_size = __len + 1;
__cs = static_cast<char*>(__builtin_alloca(__cs_size));
__len = std::__convert_from_v(_S_get_c_locale(), __cs, __cs_size,
"%.*Lf", 0, __units);
}
string_type __digits(__len, char_type());
__ctype.widen(__cs, __cs + __len, &__digits[0]);
return __intl ? _M_insert<true>(__s, __io, __fill, __digits)
: _M_insert<false>(__s, __io, __fill, __digits);
}
template<typename _CharT, typename _OutIter>
_OutIter
money_put<_CharT, _OutIter>::
do_put(iter_type __s, bool __intl, ios_base& __io, char_type __fill,
const string_type& __digits) const
{ return __intl ? _M_insert<true>(__s, __io, __fill, __digits)
: _M_insert<false>(__s, __io, __fill, __digits); }
}
template<typename _CharT, typename _InIter>
time_base::dateorder
time_get<_CharT, _InIter>::do_date_order() const
{ return time_base::no_order; }
template<typename _CharT, typename _InIter>
_InIter
time_get<_CharT, _InIter>::
_M_extract_via_format(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm,
const _CharT* __format,
__time_get_state &__state) const
{
const locale& __loc = __io._M_getloc();
const __timepunct<_CharT>& __tp = use_facet<__timepunct<_CharT> >(__loc);
const ctype<_CharT>& __ctype = use_facet<ctype<_CharT> >(__loc);
const size_t __len = char_traits<_CharT>::length(__format);
ios_base::iostate __tmperr = ios_base::goodbit;
size_t __i = 0;
for (; __beg != __end && __i < __len && !__tmperr; ++__i)
{
if (__ctype.narrow(__format[__i], 0) == '%')
{
char __c = __ctype.narrow(__format[++__i], 0);
int __mem = 0;
if (__c == 'E' || __c == 'O')
__c = __ctype.narrow(__format[++__i], 0);
switch (__c)
{
const char* __cs;
_CharT __wcs[10];
case 'a':
case 'A':
const char_type* __days[14];
__tp._M_days(&__days[0]);
__tp._M_days_abbreviated(&__days[7]);
__beg = _M_extract_name(__beg, __end, __mem, __days,
14, __io, __tmperr);
if (!__tmperr)
{
__tm->tm_wday = __mem % 7;
__state._M_have_wday = 1;
}
break;
case 'h':
case 'b':
case 'B':
const char_type* __months[24];
__tp._M_months(&__months[0]);
__tp._M_months_abbreviated(&__months[12]);
__beg = _M_extract_name(__beg, __end, __mem,
__months, 24, __io, __tmperr);
if (!__tmperr)
{
__tm->tm_mon = __mem % 12;
__state._M_have_mon = 1;
__state._M_want_xday = 1;
}
break;
case 'c':
const char_type* __dt[2];
__tp._M_date_time_formats(__dt);
__beg = _M_extract_via_format(__beg, __end, __io, __tmperr,
__tm, __dt[0], __state);
if (!__tmperr)
__state._M_want_xday = 1;
break;
case 'C':
__beg = _M_extract_num(__beg, __end, __mem, 0, 99, 2,
__io, __tmperr);
if (!__tmperr)
{
__state._M_century = __mem;
__state._M_have_century = 1;
__state._M_want_xday = 1;
}
break;
case 'd':
case 'e':
if (__ctype.is(ctype_base::space, *__beg))
++__beg;
__beg = _M_extract_num(__beg, __end, __mem, 1, 31, 2,
__io, __tmperr);
if (!__tmperr)
{
__tm->tm_mday = __mem;
__state._M_have_mday = 1;
__state._M_want_xday = 1;
}
break;
case 'D':
__cs = "%m/%d/%y";
__ctype.widen(__cs, __cs + 9, __wcs);
__beg = _M_extract_via_format(__beg, __end, __io, __tmperr,
__tm, __wcs, __state);
if (!__tmperr)
__state._M_want_xday = 1;
break;
case 'H':
__beg = _M_extract_num(__beg, __end, __mem, 0, 23, 2,
__io, __tmperr);
if (!__tmperr)
{
__tm->tm_hour = __mem;
__state._M_have_I = 0;
}
break;
case 'I':
__beg = _M_extract_num(__beg, __end, __mem, 1, 12, 2,
__io, __tmperr);
if (!__tmperr)
{
__tm->tm_hour = __mem % 12;
__state._M_have_I = 1;
}
break;
case 'j':
__beg = _M_extract_num(__beg, __end, __mem, 1, 366, 3,
__io, __tmperr);
if (!__tmperr)
{
__tm->tm_yday = __mem - 1;
__state._M_have_yday = 1;
}
break;
case 'm':
__beg = _M_extract_num(__beg, __end, __mem, 1, 12, 2,
__io, __tmperr);
if (!__tmperr)
{
__tm->tm_mon = __mem - 1;
__state._M_have_mon = 1;
}
break;
case 'M':
__beg = _M_extract_num(__beg, __end, __mem, 0, 59, 2,
__io, __tmperr);
if (!__tmperr)
__tm->tm_min = __mem;
break;
case 'n':
case 't':
while (__beg != __end
&& __ctype.is(ctype_base::space, *__beg))
++__beg;
break;
case 'p':
const char_type* __ampm[2];
__tp._M_am_pm(&__ampm[0]);
if (!__ampm[0][0] || !__ampm[1][0])
break;
__beg = _M_extract_name(__beg, __end, __mem, __ampm,
2, __io, __tmperr);
if (!__tmperr && __mem)
__state._M_is_pm = 1;
break;
case 'r':
const char_type* __ampm_format;
__tp._M_am_pm_format(&__ampm_format);
__beg = _M_extract_via_format(__beg, __end, __io, __tmperr,
__tm, __ampm_format, __state);
break;
case 'R':
__cs = "%H:%M";
__ctype.widen(__cs, __cs + 6, __wcs);
__beg = _M_extract_via_format(__beg, __end, __io, __tmperr,
__tm, __wcs, __state);
break;
case 'S':
__beg = _M_extract_num(__beg, __end, __mem, 0, 60, 2,
__io, __tmperr);
if (!__tmperr)
__tm->tm_sec = __mem;
break;
case 'T':
__cs = "%H:%M:%S";
__ctype.widen(__cs, __cs + 9, __wcs);
__beg = _M_extract_via_format(__beg, __end, __io, __tmperr,
__tm, __wcs, __state);
break;
case 'U':
__beg = _M_extract_num(__beg, __end, __mem, 0, 53, 2,
__io, __tmperr);
if (!__tmperr)
{
__state._M_week_no = __mem;
__state._M_have_uweek = 1;
}
break;
case 'w':
__beg = _M_extract_num(__beg, __end, __mem, 0, 6, 1,
__io, __tmperr);
if (!__tmperr)
{
__tm->tm_wday = __mem;
__state._M_have_wday = 1;
}
break;
case 'W':
__beg = _M_extract_num(__beg, __end, __mem, 0, 53, 2,
__io, __tmperr);
if (!__tmperr)
{
__state._M_week_no = __mem;
__state._M_have_wweek = 1;
}
break;
case 'x':
const char_type* __dates[2];
__tp._M_date_formats(__dates);
__beg = _M_extract_via_format(__beg, __end, __io, __tmperr,
__tm, __dates[0], __state);
break;
case 'X':
const char_type* __times[2];
__tp._M_time_formats(__times);
__beg = _M_extract_via_format(__beg, __end, __io, __tmperr,
__tm, __times[0], __state);
break;
case 'y':
__beg = _M_extract_num(__beg, __end, __mem, 0, 99, 2,
__io, __tmperr);
if (!__tmperr)
{
__state._M_want_century = 1;
__state._M_want_xday = 1;
__c = 0;
if (__beg != __end)
__c = __ctype.narrow(*__beg, '*');
if (__c >= '0' && __c <= '9')
{
++__beg;
__mem = __mem * 10 + (__c - '0');
if (__beg != __end)
{
__c = __ctype.narrow(*__beg, '*');
if (__c >= '0' && __c <= '9')
{
++__beg;
__mem = __mem * 10 + (__c - '0');
}
}
__mem -= 1900;
__state._M_want_century = 0;
}
else if (__mem < 69)
__mem += 100;
__tm->tm_year = __mem;
}
break;
case 'Y':
__beg = _M_extract_num(__beg, __end, __mem, 0, 9999, 4,
__io, __tmperr);
if (!__tmperr)
{
__tm->tm_year = __mem - 1900;
__state._M_want_century = 0;
__state._M_want_xday = 1;
}
break;
case 'Z':
if (__ctype.is(ctype_base::upper, *__beg))
{
int __tmp;
__beg = _M_extract_name(__beg, __end, __tmp,
__timepunct_cache<_CharT>::_S_timezones,
14, __io, __tmperr);
if (__beg != __end && !__tmperr && __tmp == 0
&& (*__beg == __ctype.widen('-')
|| *__beg == __ctype.widen('+')))
{
__beg = _M_extract_num(__beg, __end, __tmp, 0, 23, 2,
__io, __tmperr);
__beg = _M_extract_num(__beg, __end, __tmp, 0, 59, 2,
__io, __tmperr);
}
}
else
__tmperr |= ios_base::failbit;
break;
case '%':
if (*__beg == __ctype.widen('%'))
++__beg;
else
__tmperr |= ios_base::failbit;
break;
default:
__tmperr |= ios_base::failbit;
}
}
else if (__ctype.is(ctype_base::space, __format[__i]))
{
while (__beg != __end
&& __ctype.is(ctype_base::space, *__beg))
++__beg;
}
else
{
if (__ctype.tolower(__format[__i]) == __ctype.tolower(*__beg)
|| __ctype.toupper(__format[__i]) == __ctype.toupper(*__beg))
++__beg;
else
__tmperr |= ios_base::failbit;
}
}
if (__tmperr || __i != __len)
__err |= ios_base::failbit;
return __beg;
}
template<typename _CharT, typename _InIter>
_InIter
time_get<_CharT, _InIter>::
_M_extract_via_format(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm,
const _CharT* __format) const
{
__time_get_state __state = __time_get_state();
return _M_extract_via_format(__beg, __end, __io, __err, __tm,
__format, __state);
}
template<typename _CharT, typename _InIter>
_InIter
time_get<_CharT, _InIter>::
_M_extract_num(iter_type __beg, iter_type __end, int& __member,
int __min, int __max, size_t __len,
ios_base& __io, ios_base::iostate& __err) const
{
const locale& __loc = __io._M_getloc();
const ctype<_CharT>& __ctype = use_facet<ctype<_CharT> >(__loc);
size_t __i = 0;
int __value = 0;
for (; __beg != __end && __i < __len; ++__beg, (void)++__i)
{
const char __c = __ctype.narrow(*__beg, '*');
if (__c >= '0' && __c <= '9')
{
__value = __value * 10 + (__c - '0');
if (__value > __max)
break;
}
else
break;
}
if (__i && __value >= __min && __value <= __max)
__member = __value;
else
__err |= ios_base::failbit;
return __beg;
}
template<typename _CharT, typename _InIter>
_InIter
time_get<_CharT, _InIter>::
_M_extract_name(iter_type __beg, iter_type __end, int& __member,
const _CharT** __names, size_t __indexlen,
ios_base& __io, ios_base::iostate& __err) const
{
typedef char_traits<_CharT> __traits_type;
const locale& __loc = __io._M_getloc();
const ctype<_CharT>& __ctype = use_facet<ctype<_CharT> >(__loc);
size_t* __matches
= static_cast<size_t*>(__builtin_alloca(2 * sizeof(size_t)
* __indexlen));
size_t* __lengths = __matches + __indexlen;
size_t __nmatches = 0;
size_t __pos = 0;
bool __testvalid = true;
const char_type* __name;
bool __begupdated = false;
if (__beg != __end)
{
const char_type __c = *__beg;
const char_type __cl = __ctype.tolower(__c);
const char_type __cu = __ctype.toupper(__c);
for (size_t __i1 = 0; __i1 < __indexlen; ++__i1)
if (__cl == __ctype.tolower(__names[__i1][0])
|| __cu == __ctype.toupper(__names[__i1][0]))
{
__lengths[__nmatches]
= __traits_type::length(__names[__i1]);
__matches[__nmatches++] = __i1;
}
}
while (__nmatches > 1)
{
size_t __minlen = __lengths[0];
for (size_t __i2 = 1; __i2 < __nmatches; ++__i2)
__minlen = std::min(__minlen, __lengths[__i2]);
++__pos;
++__beg;
if (__pos == __minlen)
{
bool __match_longer = false;
if (__beg != __end)
{
const char_type __cl = __ctype.tolower(*__beg);
const char_type __cu = __ctype.toupper(*__beg);
for (size_t __i3 = 0; __i3 < __nmatches; ++__i3)
{
__name = __names[__matches[__i3]];
if (__lengths[__i3] > __pos
&& (__ctype.tolower(__name[__pos]) == __cl
|| __ctype.toupper(__name[__pos]) == __cu))
{
__match_longer = true;
break;
}
}
}
for (size_t __i4 = 0; __i4 < __nmatches;)
if (__match_longer == (__lengths[__i4] == __pos))
{
__matches[__i4] = __matches[--__nmatches];
__lengths[__i4] = __lengths[__nmatches];
}
else
++__i4;
if (__match_longer)
{
__minlen = __lengths[0];
for (size_t __i5 = 1; __i5 < __nmatches; ++__i5)
__minlen = std::min(__minlen, __lengths[__i5]);
}
else
{
if (__nmatches == 2 && (__indexlen & 1) == 0)
{
if (__matches[0] < __indexlen / 2)
{
if (__matches[1] == __matches[0] + __indexlen / 2)
__nmatches = 1;
}
else if (__matches[1] == __matches[0] - __indexlen / 2)
{
__matches[0] = __matches[1];
__lengths[0] = __lengths[1];
__nmatches = 1;
}
}
__begupdated = true;
break;
}
}
if (__pos < __minlen && __beg != __end)
{
const char_type __cl = __ctype.tolower(*__beg);
const char_type __cu = __ctype.toupper(*__beg);
for (size_t __i6 = 0; __i6 < __nmatches;)
{
__name = __names[__matches[__i6]];
if (__ctype.tolower(__name[__pos]) != __cl
&& __ctype.toupper(__name[__pos]) != __cu)
{
__matches[__i6] = __matches[--__nmatches];
__lengths[__i6] = __lengths[__nmatches];
}
else
++__i6;
}
}
else
break;
}
if (__nmatches == 1)
{
if (!__begupdated)
{
++__beg;
++__pos;
}
__name = __names[__matches[0]];
const size_t __len = __lengths[0];
while (__pos < __len
&& __beg != __end
&& (__ctype.tolower(__name[__pos]) == __ctype.tolower(*__beg)
|| (__ctype.toupper(__name[__pos])
== __ctype.toupper(*__beg))))
++__beg, (void)++__pos;
if (__len == __pos)
__member = __matches[0];
else
__testvalid = false;
}
else
__testvalid = false;
if (!__testvalid)
__err |= ios_base::failbit;
return __beg;
}
template<typename _CharT, typename _InIter>
_InIter
time_get<_CharT, _InIter>::
_M_extract_wday_or_month(iter_type __beg, iter_type __end, int& __member,
const _CharT** __names, size_t __indexlen,
ios_base& __io, ios_base::iostate& __err) const
{
typedef char_traits<_CharT> __traits_type;
const locale& __loc = __io._M_getloc();
const ctype<_CharT>& __ctype = use_facet<ctype<_CharT> >(__loc);
int* __matches = static_cast<int*>(__builtin_alloca(2 * sizeof(int)
* __indexlen));
size_t __nmatches = 0;
size_t* __matches_lengths = 0;
size_t __pos = 0;
if (__beg != __end)
{
const char_type __c = *__beg;
for (size_t __i = 0; __i < 2 * __indexlen; ++__i)
if (__c == __names[__i][0]
|| __c == __ctype.toupper(__names[__i][0]))
__matches[__nmatches++] = __i;
}
if (__nmatches)
{
++__beg;
++__pos;
__matches_lengths
= static_cast<size_t*>(__builtin_alloca(sizeof(size_t)
* __nmatches));
for (size_t __i = 0; __i < __nmatches; ++__i)
__matches_lengths[__i]
= __traits_type::length(__names[__matches[__i]]);
}
for (; __beg != __end; ++__beg, (void)++__pos)
{
size_t __nskipped = 0;
const char_type __c = *__beg;
for (size_t __i = 0; __i < __nmatches;)
{
const char_type* __name = __names[__matches[__i]];
if (__pos >= __matches_lengths[__i])
++__nskipped, ++__i;
else if (!(__name[__pos] == __c))
{
--__nmatches;
__matches[__i] = __matches[__nmatches];
__matches_lengths[__i] = __matches_lengths[__nmatches];
}
else
++__i;
}
if (__nskipped == __nmatches)
break;
}
if ((__nmatches == 1 && __matches_lengths[0] == __pos)
|| (__nmatches == 2 && (__matches_lengths[0] == __pos
|| __matches_lengths[1] == __pos)))
__member = (__matches[0] >= (int)__indexlen
? __matches[0] - (int)__indexlen : __matches[0]);
else
__err |= ios_base::failbit;
return __beg;
}
template<typename _CharT, typename _InIter>
_InIter
time_get<_CharT, _InIter>::
do_get_time(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm) const
{
const locale& __loc = __io._M_getloc();
const __timepunct<_CharT>& __tp = use_facet<__timepunct<_CharT> >(__loc);
const char_type* __times[2];
__tp._M_time_formats(__times);
__time_get_state __state = __time_get_state();
__beg = _M_extract_via_format(__beg, __end, __io, __err,
__tm, __times[0], __state);
__state._M_finalize_state(__tm);
if (__beg == __end)
__err |= ios_base::eofbit;
return __beg;
}
template<typename _CharT, typename _InIter>
_InIter
time_get<_CharT, _InIter>::
do_get_date(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm) const
{
const locale& __loc = __io._M_getloc();
const __timepunct<_CharT>& __tp = use_facet<__timepunct<_CharT> >(__loc);
const char_type* __dates[2];
__tp._M_date_formats(__dates);
__time_get_state __state = __time_get_state();
__beg = _M_extract_via_format(__beg, __end, __io, __err,
__tm, __dates[0], __state);
__state._M_finalize_state(__tm);
if (__beg == __end)
__err |= ios_base::eofbit;
return __beg;
}
template<typename _CharT, typename _InIter>
_InIter
time_get<_CharT, _InIter>::
do_get_weekday(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm) const
{
const locale& __loc = __io._M_getloc();
const __timepunct<_CharT>& __tp = use_facet<__timepunct<_CharT> >(__loc);
const char_type* __days[14];
__tp._M_days_abbreviated(__days);
__tp._M_days(__days + 7);
int __tmpwday;
ios_base::iostate __tmperr = ios_base::goodbit;
__beg = _M_extract_wday_or_month(__beg, __end, __tmpwday, __days, 7,
__io, __tmperr);
if (!__tmperr)
__tm->tm_wday = __tmpwday;
else
__err |= ios_base::failbit;
if (__beg == __end)
__err |= ios_base::eofbit;
return __beg;
}
template<typename _CharT, typename _InIter>
_InIter
time_get<_CharT, _InIter>::
do_get_monthname(iter_type __beg, iter_type __end,
ios_base& __io, ios_base::iostate& __err, tm* __tm) const
{
const locale& __loc = __io._M_getloc();
const __timepunct<_CharT>& __tp = use_facet<__timepunct<_CharT> >(__loc);
const char_type* __months[24];
__tp._M_months_abbreviated(__months);
__tp._M_months(__months + 12);
int __tmpmon;
ios_base::iostate __tmperr = ios_base::goodbit;
__beg = _M_extract_wday_or_month(__beg, __end, __tmpmon, __months, 12,
__io, __tmperr);
if (!__tmperr)
__tm->tm_mon = __tmpmon;
else
__err |= ios_base::failbit;
if (__beg == __end)
__err |= ios_base::eofbit;
return __beg;
}
template<typename _CharT, typename _InIter>
_InIter
time_get<_CharT, _InIter>::
do_get_year(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm) const
{
int __tmpyear;
ios_base::iostate __tmperr = ios_base::goodbit;
const locale& __loc = __io._M_getloc();
const ctype<_CharT>& __ctype = use_facet<ctype<_CharT> >(__loc);
__beg = _M_extract_num(__beg, __end, __tmpyear, 0, 99, 2,
__io, __tmperr);
if (!__tmperr)
{
char __c = 0;
if (__beg != __end)
__c = __ctype.narrow(*__beg, '*');
if (__c >= '0' && __c <= '9')
{
++__beg;
__tmpyear = __tmpyear * 10 + (__c - '0');
if (__beg != __end)
{
__c = __ctype.narrow(*__beg, '*');
if (__c >= '0' && __c <= '9')
{
++__beg;
__tmpyear = __tmpyear * 10 + (__c - '0');
}
}
__tmpyear -= 1900;
}
else if (__tmpyear < 69)
__tmpyear += 100;
__tm->tm_year = __tmpyear;
}
else
__err |= ios_base::failbit;
if (__beg == __end)
__err |= ios_base::eofbit;
return __beg;
}
template<typename _CharT, typename _InIter>
inline
_InIter
time_get<_CharT, _InIter>::
get(iter_type __s, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm, const char_type* __fmt,
const char_type* __fmtend) const
{
const locale& __loc = __io._M_getloc();
ctype<_CharT> const& __ctype = use_facet<ctype<_CharT> >(__loc);
__err = ios_base::goodbit;
bool __use_state = false;
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpmf-conversions"
if ((void*)(this->*(&time_get::do_get)) == (void*)(&time_get::do_get))
__use_state = true;
#pragma GCC diagnostic pop
__time_get_state __state = __time_get_state();
while (__fmt != __fmtend &&
__err == ios_base::goodbit)
{
if (__s == __end)
{
__err = ios_base::eofbit | ios_base::failbit;
break;
}
else if (__ctype.narrow(*__fmt, 0) == '%')
{
const char_type* __fmt_start = __fmt;
char __format;
char __mod = 0;
if (++__fmt == __fmtend)
{
__err = ios_base::failbit;
break;
}
const char __c = __ctype.narrow(*__fmt, 0);
if (__c != 'E' && __c != 'O')
__format = __c;
else if (++__fmt != __fmtend)
{
__mod = __c;
__format = __ctype.narrow(*__fmt, 0);
}
else
{
__err = ios_base::failbit;
break;
}
if (__use_state)
{
char_type __new_fmt[4];
__new_fmt[0] = __fmt_start[0];
__new_fmt[1] = __fmt_start[1];
if (__mod)
{
__new_fmt[2] = __fmt_start[2];
__new_fmt[3] = char_type();
}
else
__new_fmt[2] = char_type();
__s = _M_extract_via_format(__s, __end, __io, __err, __tm,
__new_fmt, __state);
if (__s == __end)
__err |= ios_base::eofbit;
}
else
__s = this->do_get(__s, __end, __io, __err, __tm, __format,
__mod);
++__fmt;
}
else if (__ctype.is(ctype_base::space, *__fmt))
{
++__fmt;
while (__fmt != __fmtend &&
__ctype.is(ctype_base::space, *__fmt))
++__fmt;
while (__s != __end &&
__ctype.is(ctype_base::space, *__s))
++__s;
}
else if (__ctype.tolower(*__s) == __ctype.tolower(*__fmt) ||
__ctype.toupper(*__s) == __ctype.toupper(*__fmt))
{
++__s;
++__fmt;
}
else
{
__err = ios_base::failbit;
break;
}
}
if (__use_state)
__state._M_finalize_state(__tm);
return __s;
}
template<typename _CharT, typename _InIter>
inline
_InIter
time_get<_CharT, _InIter>::
do_get(iter_type __beg, iter_type __end, ios_base& __io,
ios_base::iostate& __err, tm* __tm,
char __format, char __mod) const
{
const locale& __loc = __io._M_getloc();
ctype<_CharT> const& __ctype = use_facet<ctype<_CharT> >(__loc);
__err = ios_base::goodbit;
char_type __fmt[4];
__fmt[0] = __ctype.widen('%');
if (!__mod)
{
__fmt[1] = __format;
__fmt[2] = char_type();
}
else
{
__fmt[1] = __mod;
__fmt[2] = __format;
__fmt[3] = char_type();
}
__time_get_state __state = __time_get_state();
__beg = _M_extract_via_format(__beg, __end, __io, __err, __tm, __fmt,
__state);
__state._M_finalize_state(__tm);
if (__beg == __end)
__err |= ios_base::eofbit;
return __beg;
}
template<typename _CharT, typename _OutIter>
_OutIter
time_put<_CharT, _OutIter>::
put(iter_type __s, ios_base& __io, char_type __fill, const tm* __tm,
const _CharT* __beg, const _CharT* __end) const
{
const locale& __loc = __io._M_getloc();
ctype<_CharT> const& __ctype = use_facet<ctype<_CharT> >(__loc);
for (; __beg != __end; ++__beg)
if (__ctype.narrow(*__beg, 0) != '%')
{
*__s = *__beg;
++__s;
}
else if (++__beg != __end)
{
char __format;
char __mod = 0;
const char __c = __ctype.narrow(*__beg, 0);
if (__c != 'E' && __c != 'O')
__format = __c;
else if (++__beg != __end)
{
__mod = __c;
__format = __ctype.narrow(*__beg, 0);
}
else
break;
__s = this->do_put(__s, __io, __fill, __tm, __format, __mod);
}
else
break;
return __s;
}
template<typename _CharT, typename _OutIter>
_OutIter
time_put<_CharT, _OutIter>::
do_put(iter_type __s, ios_base& __io, char_type, const tm* __tm,
char __format, char __mod) const
{
const locale& __loc = __io._M_getloc();
ctype<_CharT> const& __ctype = use_facet<ctype<_CharT> >(__loc);
__timepunct<_CharT> const& __tp = use_facet<__timepunct<_CharT> >(__loc);
const size_t __maxlen = 128;
char_type __res[__maxlen];
char_type __fmt[4];
__fmt[0] = __ctype.widen('%');
if (!__mod)
{
__fmt[1] = __format;
__fmt[2] = char_type();
}
else
{
__fmt[1] = __mod;
__fmt[2] = __format;
__fmt[3] = char_type();
}
__tp._M_put(__res, __maxlen, __fmt, __tm);
return std::__write(__s, __res, char_traits<char_type>::length(__res));
}
extern template class moneypunct<char, false>;
extern template class moneypunct<char, true>;
extern template class moneypunct_byname<char, false>;
extern template class moneypunct_byname<char, true>;
extern template class __cxx11:: money_get<char>;
extern template class __cxx11:: money_put<char>;
extern template class __timepunct<char>;
extern template class time_put<char>;
extern template class time_put_byname<char>;
extern template class time_get<char>;
extern template class time_get_byname<char>;
extern template class messages<char>;
extern template class messages_byname<char>;
extern template
const moneypunct<char, true>*
__try_use_facet<moneypunct<char, true> >(const locale&) noexcept;
extern template
const moneypunct<char, false>*
__try_use_facet<moneypunct<char, false> >(const locale&) noexcept;
extern template
const money_put<char>*
__try_use_facet<money_put<char> >(const locale&) noexcept;
extern template
const money_get<char>*
__try_use_facet<money_get<char> >(const locale&) noexcept;
extern template
const __timepunct<char>*
__try_use_facet<__timepunct<char> >(const locale&) noexcept;
extern template
const time_put<char>*
__try_use_facet<time_put<char> >(const locale&) noexcept;
extern template
const time_get<char>*
__try_use_facet<time_get<char> >(const locale&) noexcept;
extern template
const messages<char>*
__try_use_facet<messages<char> >(const locale&) noexcept;
extern template
const moneypunct<char, true>&
use_facet<moneypunct<char, true> >(const locale&);
extern template
const moneypunct<char, false>&
use_facet<moneypunct<char, false> >(const locale&);
extern template
const money_put<char>&
use_facet<money_put<char> >(const locale&);
extern template
const money_get<char>&
use_facet<money_get<char> >(const locale&);
extern template
const __timepunct<char>&
use_facet<__timepunct<char> >(const locale&);
extern template
const time_put<char>&
use_facet<time_put<char> >(const locale&);
extern template
const time_get<char>&
use_facet<time_get<char> >(const locale&);
extern template
const messages<char>&
use_facet<messages<char> >(const locale&);
extern template
bool
has_facet<moneypunct<char> >(const locale&);
extern template
bool
has_facet<money_put<char> >(const locale&);
extern template
bool
has_facet<money_get<char> >(const locale&);
extern template
bool
has_facet<__timepunct<char> >(const locale&);
extern template
bool
has_facet<time_put<char> >(const locale&);
extern template
bool
has_facet<time_get<char> >(const locale&);
extern template
bool
has_facet<messages<char> >(const locale&);
extern template class moneypunct<wchar_t, false>;
extern template class moneypunct<wchar_t, true>;
extern template class moneypunct_byname<wchar_t, false>;
extern template class moneypunct_byname<wchar_t, true>;
extern template class __cxx11:: money_get<wchar_t>;
extern template class __cxx11:: money_put<wchar_t>;
extern template class __timepunct<wchar_t>;
extern template class time_put<wchar_t>;
extern template class time_put_byname<wchar_t>;
extern template class time_get<wchar_t>;
extern template class time_get_byname<wchar_t>;
extern template class messages<wchar_t>;
extern template class messages_byname<wchar_t>;
extern template
const moneypunct<wchar_t, true>*
__try_use_facet<moneypunct<wchar_t, true> >(const locale&) noexcept;
extern template
const moneypunct<wchar_t, false>*
__try_use_facet<moneypunct<wchar_t, false> >(const locale&) noexcept;
extern template
const money_put<wchar_t>*
__try_use_facet<money_put<wchar_t> >(const locale&) noexcept;
extern template
const money_get<wchar_t>*
__try_use_facet<money_get<wchar_t> >(const locale&) noexcept;
extern template
const __timepunct<wchar_t>*
__try_use_facet<__timepunct<wchar_t> >(const locale&) noexcept;
extern template
const time_put<wchar_t>*
__try_use_facet<time_put<wchar_t> >(const locale&) noexcept;
extern template
const time_get<wchar_t>*
__try_use_facet<time_get<wchar_t> >(const locale&) noexcept;
extern template
const messages<wchar_t>*
__try_use_facet<messages<wchar_t> >(const locale&) noexcept;
extern template
const moneypunct<wchar_t, true>&
use_facet<moneypunct<wchar_t, true> >(const locale&);
extern template
const moneypunct<wchar_t, false>&
use_facet<moneypunct<wchar_t, false> >(const locale&);
extern template
const money_put<wchar_t>&
use_facet<money_put<wchar_t> >(const locale&);
extern template
const money_get<wchar_t>&
use_facet<money_get<wchar_t> >(const locale&);
extern template
const __timepunct<wchar_t>&
use_facet<__timepunct<wchar_t> >(const locale&);
extern template
const time_put<wchar_t>&
use_facet<time_put<wchar_t> >(const locale&);
extern template
const time_get<wchar_t>&
use_facet<time_get<wchar_t> >(const locale&);
extern template
const messages<wchar_t>&
use_facet<messages<wchar_t> >(const locale&);
extern template
bool
has_facet<moneypunct<wchar_t> >(const locale&);
extern template
bool
has_facet<money_put<wchar_t> >(const locale&);
extern template
bool
has_facet<money_get<wchar_t> >(const locale&);
extern template
bool
has_facet<__timepunct<wchar_t> >(const locale&);
extern template
bool
has_facet<time_put<wchar_t> >(const locale&);
extern template
bool
has_facet<time_get<wchar_t> >(const locale&);
extern template
bool
has_facet<messages<wchar_t> >(const locale&);
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _OutStr, typename _InChar, typename _Codecvt,
typename _State, typename _Fn>
bool
__do_str_codecvt(const _InChar* __first, const _InChar* __last,
_OutStr& __outstr, const _Codecvt& __cvt, _State& __state,
size_t& __count, _Fn __fn)
{
if (__first == __last)
{
__outstr.clear();
__count = 0;
return true;
}
size_t __outchars = 0;
auto __next = __first;
const auto __maxlen = __cvt.max_length() + 1;
codecvt_base::result __result;
do
{
__outstr.resize(__outstr.size() + (__last - __next) * __maxlen);
auto __outnext = &__outstr.front() + __outchars;
auto const __outlast = &__outstr.back() + 1;
__result = (__cvt.*__fn)(__state, __next, __last, __next,
__outnext, __outlast, __outnext);
__outchars = __outnext - &__outstr.front();
}
while (__result == codecvt_base::partial && __next != __last
&& ptrdiff_t(__outstr.size() - __outchars) < __maxlen);
if (__result == codecvt_base::error)
{
__count = __next - __first;
return false;
}
if constexpr (is_same<typename _Codecvt::intern_type,
typename _Codecvt::extern_type>())
if (__result == codecvt_base::noconv)
{
__outstr.assign(__first, __last);
__count = __last - __first;
return true;
}
__outstr.resize(__outchars);
__count = __next - __first;
return true;
}
template<typename _CharT, typename _Traits, typename _Alloc, typename _State>
inline bool
__str_codecvt_in(const char* __first, const char* __last,
basic_string<_CharT, _Traits, _Alloc>& __outstr,
const codecvt<_CharT, char, _State>& __cvt,
_State& __state, size_t& __count)
{
using _Codecvt = codecvt<_CharT, char, _State>;
using _ConvFn
= codecvt_base::result
(_Codecvt::*)(_State&, const char*, const char*, const char*&,
_CharT*, _CharT*, _CharT*&) const;
_ConvFn __fn = &codecvt<_CharT, char, _State>::in;
return __do_str_codecvt(__first, __last, __outstr, __cvt, __state,
__count, __fn);
}
template<typename _CharT, typename _Traits, typename _Alloc, typename _State>
inline bool
__str_codecvt_in(const char* __first, const char* __last,
basic_string<_CharT, _Traits, _Alloc>& __outstr,
const codecvt<_CharT, char, _State>& __cvt)
{
_State __state = {};
size_t __n;
return __str_codecvt_in(__first, __last, __outstr, __cvt, __state, __n);
}
template<typename _CharT, typename _Traits, typename _Alloc, typename _State>
inline bool
__str_codecvt_in_all(const char* __first, const char* __last,
basic_string<_CharT, _Traits, _Alloc>& __outstr,
const codecvt<_CharT, char, _State>& __cvt)
{
_State __state = {};
size_t __n;
return __str_codecvt_in(__first, __last, __outstr, __cvt, __state, __n)
&& (__n == size_t(__last - __first));
}
template<typename _CharT, typename _Traits, typename _Alloc, typename _State>
inline bool
__str_codecvt_out(const _CharT* __first, const _CharT* __last,
basic_string<char, _Traits, _Alloc>& __outstr,
const codecvt<_CharT, char, _State>& __cvt,
_State& __state, size_t& __count)
{
using _Codecvt = codecvt<_CharT, char, _State>;
using _ConvFn
= codecvt_base::result
(_Codecvt::*)(_State&, const _CharT*, const _CharT*, const _CharT*&,
char*, char*, char*&) const;
_ConvFn __fn = &codecvt<_CharT, char, _State>::out;
return __do_str_codecvt(__first, __last, __outstr, __cvt, __state,
__count, __fn);
}
template<typename _CharT, typename _Traits, typename _Alloc, typename _State>
inline bool
__str_codecvt_out(const _CharT* __first, const _CharT* __last,
basic_string<char, _Traits, _Alloc>& __outstr,
const codecvt<_CharT, char, _State>& __cvt)
{
_State __state = {};
size_t __n;
return __str_codecvt_out(__first, __last, __outstr, __cvt, __state, __n);
}
template<typename _CharT, typename _Traits, typename _Alloc, typename _State>
inline bool
__str_codecvt_out_all(const _CharT* __first, const _CharT* __last,
basic_string<char, _Traits, _Alloc>& __outstr,
const codecvt<_CharT, char, _State>& __cvt)
{
_State __state = {};
size_t __n;
return __str_codecvt_out(__first, __last, __outstr, __cvt, __state, __n)
&& (__n == size_t(__last - __first));
}
namespace __detail
{
template<typename _Tp>
struct _Scoped_ptr
{
__attribute__((__nonnull__(2)))
explicit
_Scoped_ptr(_Tp* __ptr) noexcept
: _M_ptr(__ptr)
{ }
_Scoped_ptr(_Tp* __ptr, const char* __msg)
: _M_ptr(__ptr)
{
if (!__ptr)
__throw_logic_error(__msg);
}
~_Scoped_ptr() { delete _M_ptr; }
_Scoped_ptr(const _Scoped_ptr&) = delete;
_Scoped_ptr& operator=(const _Scoped_ptr&) = delete;
__attribute__((__returns_nonnull__))
_Tp* operator->() const noexcept { return _M_ptr; }
_Tp& operator*() const noexcept { return *_M_ptr; }
private:
_Tp* _M_ptr;
};
}
namespace __cxx11 {
template<typename _Codecvt, typename _Elem = wchar_t,
typename _Wide_alloc = allocator<_Elem>,
typename _Byte_alloc = allocator<char>>
class wstring_convert
{
public:
typedef basic_string<char, char_traits<char>, _Byte_alloc> byte_string;
typedef basic_string<_Elem, char_traits<_Elem>, _Wide_alloc> wide_string;
typedef typename _Codecvt::state_type state_type;
typedef typename wide_string::traits_type::int_type int_type;
wstring_convert() : _M_cvt(new _Codecvt()) { }
explicit
wstring_convert(_Codecvt* __pcvt) : _M_cvt(__pcvt, "wstring_convert")
{ }
wstring_convert(_Codecvt* __pcvt, state_type __state)
: _M_cvt(__pcvt, "wstring_convert"),
_M_state(__state), _M_with_cvtstate(true)
{ }
explicit
wstring_convert(const byte_string& __byte_err,
const wide_string& __wide_err = wide_string())
: _M_cvt(new _Codecvt),
_M_byte_err_string(__byte_err), _M_wide_err_string(__wide_err),
_M_with_strings(true)
{ }
~wstring_convert() = default;
wstring_convert(const wstring_convert&) = delete;
wstring_convert& operator=(const wstring_convert&) = delete;
wide_string
from_bytes(char __byte)
{
char __bytes[2] = { __byte };
return from_bytes(__bytes, __bytes+1);
}
wide_string
from_bytes(const char* __ptr)
{ return from_bytes(__ptr, __ptr+char_traits<char>::length(__ptr)); }
wide_string
from_bytes(const byte_string& __str)
{
auto __ptr = __str.data();
return from_bytes(__ptr, __ptr + __str.size());
}
wide_string
from_bytes(const char* __first, const char* __last)
{
if (!_M_with_cvtstate)
_M_state = state_type();
wide_string __out{ _M_wide_err_string.get_allocator() };
if (__str_codecvt_in(__first, __last, __out, *_M_cvt, _M_state,
_M_count))
return __out;
if (_M_with_strings)
return _M_wide_err_string;
__throw_range_error("wstring_convert::from_bytes");
}
byte_string
to_bytes(_Elem __wchar)
{
_Elem __wchars[2] = { __wchar };
return to_bytes(__wchars, __wchars+1);
}
byte_string
to_bytes(const _Elem* __ptr)
{
return to_bytes(__ptr, __ptr+wide_string::traits_type::length(__ptr));
}
byte_string
to_bytes(const wide_string& __wstr)
{
auto __ptr = __wstr.data();
return to_bytes(__ptr, __ptr + __wstr.size());
}
byte_string
to_bytes(const _Elem* __first, const _Elem* __last)
{
if (!_M_with_cvtstate)
_M_state = state_type();
byte_string __out{ _M_byte_err_string.get_allocator() };
if (__str_codecvt_out(__first, __last, __out, *_M_cvt, _M_state,
_M_count))
return __out;
if (_M_with_strings)
return _M_byte_err_string;
__throw_range_error("wstring_convert::to_bytes");
}
size_t converted() const noexcept { return _M_count; }
state_type state() const { return _M_state; }
private:
__detail::_Scoped_ptr<_Codecvt> _M_cvt;
byte_string _M_byte_err_string;
wide_string _M_wide_err_string;
state_type _M_state = state_type();
size_t _M_count = 0;
bool _M_with_cvtstate = false;
bool _M_with_strings = false;
};
}
template<typename _Codecvt, typename _Elem = wchar_t,
typename _Tr = char_traits<_Elem>>
class wbuffer_convert : public basic_streambuf<_Elem, _Tr>
{
typedef basic_streambuf<_Elem, _Tr> _Wide_streambuf;
public:
typedef typename _Codecvt::state_type state_type;
wbuffer_convert() : wbuffer_convert(nullptr) { }
explicit
wbuffer_convert(streambuf* __bytebuf, _Codecvt* __pcvt = new _Codecvt,
state_type __state = state_type())
: _M_buf(__bytebuf), _M_cvt(__pcvt, "wbuffer_convert"),
_M_state(__state), _M_always_noconv(_M_cvt->always_noconv())
{
if (_M_buf)
{
this->setp(_M_put_area, _M_put_area + _S_buffer_length);
this->setg(_M_get_area + _S_putback_length,
_M_get_area + _S_putback_length,
_M_get_area + _S_putback_length);
}
}
~wbuffer_convert() = default;
wbuffer_convert(const wbuffer_convert&) = delete;
wbuffer_convert& operator=(const wbuffer_convert&) = delete;
streambuf* rdbuf() const noexcept { return _M_buf; }
streambuf*
rdbuf(streambuf *__bytebuf) noexcept
{
auto __prev = _M_buf;
_M_buf = __bytebuf;
return __prev;
}
state_type state() const noexcept { return _M_state; }
protected:
int
sync()
{ return _M_buf && _M_conv_put() && !_M_buf->pubsync() ? 0 : -1; }
typename _Wide_streambuf::int_type
overflow(typename _Wide_streambuf::int_type __out)
{
if (!_M_buf || !_M_conv_put())
return _Tr::eof();
else if (!_Tr::eq_int_type(__out, _Tr::eof()))
return this->sputc(__out);
return _Tr::not_eof(__out);
}
typename _Wide_streambuf::int_type
underflow()
{
if (!_M_buf)
return _Tr::eof();
if (this->gptr() < this->egptr() || (_M_buf && _M_conv_get()))
return _Tr::to_int_type(*this->gptr());
else
return _Tr::eof();
}
streamsize
xsputn(const typename _Wide_streambuf::char_type* __s, streamsize __n)
{
if (!_M_buf || __n == 0)
return 0;
streamsize __done = 0;
do
{
auto __nn = std::min<streamsize>(this->epptr() - this->pptr(),
__n - __done);
_Tr::copy(this->pptr(), __s + __done, __nn);
this->pbump(__nn);
__done += __nn;
} while (__done < __n && _M_conv_put());
return __done;
}
private:
bool
_M_conv_get()
{
const streamsize __pb1 = this->gptr() - this->eback();
const streamsize __pb2 = _S_putback_length;
const streamsize __npb = std::min(__pb1, __pb2);
_Tr::move(_M_get_area + _S_putback_length - __npb,
this->gptr() - __npb, __npb);
streamsize __nbytes = sizeof(_M_get_buf) - _M_unconv;
__nbytes = std::min(__nbytes, _M_buf->in_avail());
if (__nbytes < 1)
__nbytes = 1;
__nbytes = _M_buf->sgetn(_M_get_buf + _M_unconv, __nbytes);
if (__nbytes < 1)
return false;
__nbytes += _M_unconv;
_Elem* __outbuf = _M_get_area + _S_putback_length;
_Elem* __outnext = __outbuf;
const char* __bnext = _M_get_buf;
codecvt_base::result __result;
if (_M_always_noconv)
__result = codecvt_base::noconv;
else
{
_Elem* __outend = _M_get_area + _S_buffer_length;
__result = _M_cvt->in(_M_state,
__bnext, __bnext + __nbytes, __bnext,
__outbuf, __outend, __outnext);
}
if (__result == codecvt_base::noconv)
{
auto __get_buf = reinterpret_cast<const _Elem*>(_M_get_buf);
_Tr::copy(__outbuf, __get_buf, __nbytes);
_M_unconv = 0;
return true;
}
if ((_M_unconv = _M_get_buf + __nbytes - __bnext))
char_traits<char>::move(_M_get_buf, __bnext, _M_unconv);
this->setg(__outbuf, __outbuf, __outnext);
return __result != codecvt_base::error;
}
bool
_M_put(...)
{ return false; }
bool
_M_put(const char* __p, streamsize __n)
{
if (_M_buf->sputn(__p, __n) < __n)
return false;
return true;
}
bool
_M_conv_put()
{
_Elem* const __first = this->pbase();
const _Elem* const __last = this->pptr();
const streamsize __pending = __last - __first;
if (_M_always_noconv)
return _M_put(__first, __pending);
char __outbuf[2 * _S_buffer_length];
const _Elem* __next = __first;
const _Elem* __start;
do
{
__start = __next;
char* __outnext = __outbuf;
char* const __outlast = __outbuf + sizeof(__outbuf);
auto __result = _M_cvt->out(_M_state, __next, __last, __next,
__outnext, __outlast, __outnext);
if (__result == codecvt_base::error)
return false;
else if (__result == codecvt_base::noconv)
return _M_put(__next, __pending);
if (!_M_put(__outbuf, __outnext - __outbuf))
return false;
}
while (__next != __last && __next != __start);
if (__next != __last)
_Tr::move(__first, __next, __last - __next);
this->pbump(__first - __next);
return __next != __first;
}
streambuf* _M_buf;
__detail::_Scoped_ptr<_Codecvt> _M_cvt;
state_type _M_state;
static const streamsize _S_buffer_length = 32;
static const streamsize _S_putback_length = 3;
_Elem _M_put_area[_S_buffer_length];
_Elem _M_get_area[_S_buffer_length];
streamsize _M_unconv = 0;
char _M_get_buf[_S_buffer_length-_S_putback_length];
bool _M_always_noconv;
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Facet>
inline const _Facet&
__check_facet(const _Facet* __f)
{
if (!__f)
__throw_bad_cast();
return *__f;
}
template<typename _CharT, typename _Traits>
class basic_ios : public ios_base
{
public:
typedef _CharT char_type;
typedef typename _Traits::int_type int_type;
typedef typename _Traits::pos_type pos_type;
typedef typename _Traits::off_type off_type;
typedef _Traits traits_type;
typedef ctype<_CharT> __ctype_type;
typedef num_put<_CharT, ostreambuf_iterator<_CharT, _Traits> >
__num_put_type;
typedef num_get<_CharT, istreambuf_iterator<_CharT, _Traits> >
__num_get_type;
protected:
basic_ostream<_CharT, _Traits>* _M_tie;
mutable char_type _M_fill;
mutable bool _M_fill_init;
basic_streambuf<_CharT, _Traits>* _M_streambuf;
const __ctype_type* _M_ctype;
const __num_put_type* _M_num_put;
const __num_get_type* _M_num_get;
public:
explicit operator bool() const
{ return !this->fail(); }
bool
operator!() const
{ return this->fail(); }
iostate
rdstate() const
{ return _M_streambuf_state; }
void
clear(iostate __state = goodbit);
void
setstate(iostate __state)
{ this->clear(this->rdstate() | __state); }
void
_M_setstate(iostate __state)
{
_M_streambuf_state |= __state;
if (this->exceptions() & __state)
throw;
}
bool
good() const
{ return this->rdstate() == 0; }
bool
eof() const
{ return (this->rdstate() & eofbit) != 0; }
bool
fail() const
{ return (this->rdstate() & (badbit | failbit)) != 0; }
bool
bad() const
{ return (this->rdstate() & badbit) != 0; }
iostate
exceptions() const
{ return _M_exception; }
void
exceptions(iostate __except)
{
_M_exception = __except;
this->clear(_M_streambuf_state);
}
explicit
basic_ios(basic_streambuf<_CharT, _Traits>* __sb)
: ios_base(), _M_tie(0), _M_fill(), _M_fill_init(false), _M_streambuf(0),
_M_ctype(0), _M_num_put(0), _M_num_get(0)
{ this->init(__sb); }
virtual
~basic_ios() { }
basic_ostream<_CharT, _Traits>*
tie() const
{ return _M_tie; }
basic_ostream<_CharT, _Traits>*
tie(basic_ostream<_CharT, _Traits>* __tiestr)
{
basic_ostream<_CharT, _Traits>* __old = _M_tie;
_M_tie = __tiestr;
return __old;
}
basic_streambuf<_CharT, _Traits>*
rdbuf() const
{ return _M_streambuf; }
basic_streambuf<_CharT, _Traits>*
rdbuf(basic_streambuf<_CharT, _Traits>* __sb);
basic_ios&
copyfmt(const basic_ios& __rhs);
char_type
fill() const
{
if (!_M_fill_init)
{
_M_fill = this->widen(' ');
_M_fill_init = true;
}
return _M_fill;
}
char_type
fill(char_type __ch)
{
char_type __old = this->fill();
_M_fill = __ch;
return __old;
}
locale
imbue(const locale& __loc);
char
narrow(char_type __c, char __dfault) const
{ return __check_facet(_M_ctype).narrow(__c, __dfault); }
char_type
widen(char __c) const
{ return __check_facet(_M_ctype).widen(__c); }
protected:
basic_ios()
: ios_base(), _M_tie(0), _M_fill(char_type()), _M_fill_init(false),
_M_streambuf(0), _M_ctype(0), _M_num_put(0), _M_num_get(0)
{ }
void
init(basic_streambuf<_CharT, _Traits>* __sb);
basic_ios(const basic_ios&) = delete;
basic_ios& operator=(const basic_ios&) = delete;
void
move(basic_ios& __rhs)
{
ios_base::_M_move(__rhs);
_M_cache_locale(_M_ios_locale);
this->tie(__rhs.tie(nullptr));
_M_fill = __rhs._M_fill;
_M_fill_init = __rhs._M_fill_init;
_M_streambuf = nullptr;
}
void
move(basic_ios&& __rhs)
{ this->move(__rhs); }
void
swap(basic_ios& __rhs) noexcept
{
ios_base::_M_swap(__rhs);
_M_cache_locale(_M_ios_locale);
__rhs._M_cache_locale(__rhs._M_ios_locale);
std::swap(_M_tie, __rhs._M_tie);
std::swap(_M_fill, __rhs._M_fill);
std::swap(_M_fill_init, __rhs._M_fill_init);
}
void
set_rdbuf(basic_streambuf<_CharT, _Traits>* __sb)
{ _M_streambuf = __sb; }
void
_M_cache_locale(const locale& __loc);
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _CharT, typename _Traits>
void
basic_ios<_CharT, _Traits>::clear(iostate __state)
{
if (this->rdbuf())
_M_streambuf_state = __state;
else
_M_streambuf_state = __state | badbit;
if (this->exceptions() & this->rdstate())
__throw_ios_failure(("basic_ios::clear"));
}
template<typename _CharT, typename _Traits>
basic_streambuf<_CharT, _Traits>*
basic_ios<_CharT, _Traits>::rdbuf(basic_streambuf<_CharT, _Traits>* __sb)
{
basic_streambuf<_CharT, _Traits>* __old = _M_streambuf;
_M_streambuf = __sb;
this->clear();
return __old;
}
template<typename _CharT, typename _Traits>
basic_ios<_CharT, _Traits>&
basic_ios<_CharT, _Traits>::copyfmt(const basic_ios& __rhs)
{
if (this != std::__addressof(__rhs))
{
_Words* __words = (__rhs._M_word_size <= _S_local_word_size) ?
_M_local_word : new _Words[__rhs._M_word_size];
_Callback_list* __cb = __rhs._M_callbacks;
if (__cb)
__cb->_M_add_reference();
_M_call_callbacks(erase_event);
if (_M_word != _M_local_word)
{
delete [] _M_word;
_M_word = 0;
}
_M_dispose_callbacks();
_M_callbacks = __cb;
for (int __i = 0; __i < __rhs._M_word_size; ++__i)
__words[__i] = __rhs._M_word[__i];
_M_word = __words;
_M_word_size = __rhs._M_word_size;
this->flags(__rhs.flags());
this->width(__rhs.width());
this->precision(__rhs.precision());
this->tie(__rhs.tie());
this->fill(__rhs.fill());
_M_ios_locale = __rhs.getloc();
_M_cache_locale(_M_ios_locale);
_M_call_callbacks(copyfmt_event);
this->exceptions(__rhs.exceptions());
}
return *this;
}
template<typename _CharT, typename _Traits>
locale
basic_ios<_CharT, _Traits>::imbue(const locale& __loc)
{
locale __old(this->getloc());
ios_base::imbue(__loc);
_M_cache_locale(__loc);
if (this->rdbuf() != 0)
this->rdbuf()->pubimbue(__loc);
return __old;
}
template<typename _CharT, typename _Traits>
void
basic_ios<_CharT, _Traits>::init(basic_streambuf<_CharT, _Traits>* __sb)
{
ios_base::_M_init();
_M_cache_locale(_M_ios_locale);
_M_fill = _CharT();
_M_fill_init = false;
_M_tie = 0;
_M_exception = goodbit;
_M_streambuf = __sb;
_M_streambuf_state = __sb ? goodbit : badbit;
}
template<typename _CharT, typename _Traits>
void
basic_ios<_CharT, _Traits>::_M_cache_locale(const locale& __loc)
{
_M_ctype = std::__try_use_facet<__ctype_type>(__loc);
_M_num_put = std::__try_use_facet<__num_put_type>(__loc);
_M_num_get = std::__try_use_facet<__num_get_type>(__loc);
}
extern template class basic_ios<char>;
extern template class basic_ios<wchar_t>;
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _CharT, typename _Traits>
class basic_ostream : virtual public basic_ios<_CharT, _Traits>
{
public:
typedef _CharT char_type;
typedef typename _Traits::int_type int_type;
typedef typename _Traits::pos_type pos_type;
typedef typename _Traits::off_type off_type;
typedef _Traits traits_type;
typedef basic_streambuf<_CharT, _Traits> __streambuf_type;
typedef basic_ios<_CharT, _Traits> __ios_type;
typedef basic_ostream<_CharT, _Traits> __ostream_type;
typedef num_put<_CharT, ostreambuf_iterator<_CharT, _Traits> >
__num_put_type;
typedef ctype<_CharT> __ctype_type;
explicit
basic_ostream(__streambuf_type* __sb)
{ this->init(__sb); }
virtual
~basic_ostream() { }
class sentry;
friend class sentry;
__ostream_type&
operator<<(__ostream_type& (*__pf)(__ostream_type&))
{
return __pf(*this);
}
__ostream_type&
operator<<(__ios_type& (*__pf)(__ios_type&))
{
__pf(*this);
return *this;
}
__ostream_type&
operator<<(ios_base& (*__pf) (ios_base&))
{
__pf(*this);
return *this;
}
__ostream_type&
operator<<(long __n)
{ return _M_insert(__n); }
__ostream_type&
operator<<(unsigned long __n)
{ return _M_insert(__n); }
__ostream_type&
operator<<(bool __n)
{ return _M_insert(__n); }
__ostream_type&
operator<<(short __n);
__ostream_type&
operator<<(unsigned short __n)
{
return _M_insert(static_cast<unsigned long>(__n));
}
__ostream_type&
operator<<(int __n);
__ostream_type&
operator<<(unsigned int __n)
{
return _M_insert(static_cast<unsigned long>(__n));
}
__ostream_type&
operator<<(long long __n)
{ return _M_insert(__n); }
__ostream_type&
operator<<(unsigned long long __n)
{ return _M_insert(__n); }
__ostream_type&
operator<<(double __f)
{ return _M_insert(__f); }
__ostream_type&
operator<<(float __f)
{
return _M_insert(static_cast<double>(__f));
}
__ostream_type&
operator<<(long double __f)
{ return _M_insert(__f); }
__ostream_type&
operator<<(const void* __p)
{ return _M_insert(__p); }
__ostream_type&
operator<<(nullptr_t)
{ return *this << "nullptr"; }
__ostream_type&
operator<<(__streambuf_type* __sb);
__ostream_type&
put(char_type __c);
__ostream_type&
write(const char_type* __s, streamsize __n);
__ostream_type&
flush();
pos_type
tellp();
__ostream_type&
seekp(pos_type);
__ostream_type&
seekp(off_type, ios_base::seekdir);
protected:
basic_ostream()
{ this->init(0); }
basic_ostream(basic_iostream<_CharT, _Traits>&) { }
basic_ostream(const basic_ostream&) = delete;
basic_ostream(basic_ostream&& __rhs)
: __ios_type()
{ __ios_type::move(__rhs); }
basic_ostream& operator=(const basic_ostream&) = delete;
basic_ostream&
operator=(basic_ostream&& __rhs)
{
swap(__rhs);
return *this;
}
void
swap(basic_ostream& __rhs)
{ __ios_type::swap(__rhs); }
template<typename _ValueT>
__ostream_type&
_M_insert(_ValueT __v);
private:
void
_M_write(const char_type* __s, streamsize __n)
{ std::__ostream_insert(*this, __s, __n); }
};
template <typename _CharT, typename _Traits>
class basic_ostream<_CharT, _Traits>::sentry
{
bool _M_ok;
basic_ostream<_CharT, _Traits>& _M_os;
public:
explicit
sentry(basic_ostream<_CharT, _Traits>& __os);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
~sentry()
{
if (bool(_M_os.flags() & ios_base::unitbuf) && !uncaught_exception())
{
if (_M_os.rdbuf() && _M_os.rdbuf()->pubsync() == -1)
_M_os.setstate(ios_base::badbit);
}
}
#pragma GCC diagnostic pop
explicit
operator bool() const
{ return _M_ok; }
};
template<typename _CharT, typename _Traits>
inline basic_ostream<_CharT, _Traits>&
operator<<(basic_ostream<_CharT, _Traits>& __out, _CharT __c)
{
if (__out.width() != 0)
return __ostream_insert(__out, &__c, 1);
__out.put(__c);
return __out;
}
template<typename _CharT, typename _Traits>
inline basic_ostream<_CharT, _Traits>&
operator<<(basic_ostream<_CharT, _Traits>& __out, char __c)
{ return (__out << __out.widen(__c)); }
template<typename _Traits>
inline basic_ostream<char, _Traits>&
operator<<(basic_ostream<char, _Traits>& __out, char __c)
{
if (__out.width() != 0)
return __ostream_insert(__out, &__c, 1);
__out.put(__c);
return __out;
}
template<typename _Traits>
inline basic_ostream<char, _Traits>&
operator<<(basic_ostream<char, _Traits>& __out, signed char __c)
{ return (__out << static_cast<char>(__c)); }
template<typename _Traits>
inline basic_ostream<char, _Traits>&
operator<<(basic_ostream<char, _Traits>& __out, unsigned char __c)
{ return (__out << static_cast<char>(__c)); }
template<typename _CharT, typename _Traits>
inline basic_ostream<_CharT, _Traits>&
operator<<(basic_ostream<_CharT, _Traits>& __out, const _CharT* __s)
{
if (!__s)
__out.setstate(ios_base::badbit);
else
__ostream_insert(__out, __s,
static_cast<streamsize>(_Traits::length(__s)));
return __out;
}
template<typename _CharT, typename _Traits>
basic_ostream<_CharT, _Traits> &
operator<<(basic_ostream<_CharT, _Traits>& __out, const char* __s);
template<typename _Traits>
inline basic_ostream<char, _Traits>&
operator<<(basic_ostream<char, _Traits>& __out, const char* __s)
{
if (!__s)
__out.setstate(ios_base::badbit);
else
__ostream_insert(__out, __s,
static_cast<streamsize>(_Traits::length(__s)));
return __out;
}
template<typename _Traits>
inline basic_ostream<char, _Traits>&
operator<<(basic_ostream<char, _Traits>& __out, const signed char* __s)
{ return (__out << reinterpret_cast<const char*>(__s)); }
template<typename _Traits>
inline basic_ostream<char, _Traits> &
operator<<(basic_ostream<char, _Traits>& __out, const unsigned char* __s)
{ return (__out << reinterpret_cast<const char*>(__s)); }
template<typename _CharT, typename _Traits>
inline basic_ostream<_CharT, _Traits>&
endl(basic_ostream<_CharT, _Traits>& __os)
{ return flush(__os.put(__os.widen('\n'))); }
template<typename _CharT, typename _Traits>
inline basic_ostream<_CharT, _Traits>&
ends(basic_ostream<_CharT, _Traits>& __os)
{ return __os.put(_CharT()); }
template<typename _CharT, typename _Traits>
inline basic_ostream<_CharT, _Traits>&
flush(basic_ostream<_CharT, _Traits>& __os)
{ return __os.flush(); }
template<typename _Tp>
using _Require_derived_from_ios_base
= _Require<is_class<_Tp>, __not_<is_same<_Tp, ios_base>>,
is_convertible<typename add_pointer<_Tp>::type, ios_base*>>;
template<typename _Os, typename _Tp,
typename = _Require_derived_from_ios_base<_Os>,
typename
= decltype(std::declval<_Os&>() << std::declval<const _Tp&>())>
using __rvalue_stream_insertion_t = _Os&&;
template<typename _Ostream, typename _Tp>
inline __rvalue_stream_insertion_t<_Ostream, _Tp>
operator<<(_Ostream&& __os, const _Tp& __x)
{
__os << __x;
return std::move(__os);
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _CharT, typename _Traits>
basic_ostream<_CharT, _Traits>::sentry::
sentry(basic_ostream<_CharT, _Traits>& __os)
: _M_ok(false), _M_os(__os)
{
if (__os.tie() && __os.good())
__os.tie()->flush();
if (__os.good())
_M_ok = true;
else if (__os.bad())
__os.setstate(ios_base::failbit);
}
template<typename _CharT, typename _Traits>
template<typename _ValueT>
basic_ostream<_CharT, _Traits>&
basic_ostream<_CharT, _Traits>::
_M_insert(_ValueT __v)
{
sentry __cerb(*this);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
const __num_put_type& __np = __check_facet(this->_M_num_put);
if (__np.put(*this, *this, this->fill(), __v).failed())
__err |= ios_base::badbit;
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(__err);
}
return *this;
}
template<typename _CharT, typename _Traits>
basic_ostream<_CharT, _Traits>&
basic_ostream<_CharT, _Traits>::
operator<<(short __n)
{
const ios_base::fmtflags __fmt = this->flags() & ios_base::basefield;
if (__fmt == ios_base::oct || __fmt == ios_base::hex)
return _M_insert(static_cast<long>(static_cast<unsigned short>(__n)));
else
return _M_insert(static_cast<long>(__n));
}
template<typename _CharT, typename _Traits>
basic_ostream<_CharT, _Traits>&
basic_ostream<_CharT, _Traits>::
operator<<(int __n)
{
const ios_base::fmtflags __fmt = this->flags() & ios_base::basefield;
if (__fmt == ios_base::oct || __fmt == ios_base::hex)
return _M_insert(static_cast<long>(static_cast<unsigned int>(__n)));
else
return _M_insert(static_cast<long>(__n));
}
template<typename _CharT, typename _Traits>
basic_ostream<_CharT, _Traits>&
basic_ostream<_CharT, _Traits>::
operator<<(__streambuf_type* __sbin)
{
ios_base::iostate __err = ios_base::goodbit;
sentry __cerb(*this);
if (__cerb && __sbin)
{
try
{
if (!__copy_streambufs(__sbin, this->rdbuf()))
__err |= ios_base::failbit;
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::failbit); }
}
else if (!__sbin)
__err |= ios_base::badbit;
if (__err)
this->setstate(__err);
return *this;
}
template<typename _CharT, typename _Traits>
basic_ostream<_CharT, _Traits>&
basic_ostream<_CharT, _Traits>::
put(char_type __c)
{
sentry __cerb(*this);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
const int_type __put = this->rdbuf()->sputc(__c);
if (traits_type::eq_int_type(__put, traits_type::eof()))
__err |= ios_base::badbit;
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(__err);
}
return *this;
}
template<typename _CharT, typename _Traits>
basic_ostream<_CharT, _Traits>&
basic_ostream<_CharT, _Traits>::
write(const _CharT* __s, streamsize __n)
{
sentry __cerb(*this);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
if (this->rdbuf()->sputn(__s, __n) != __n)
__err = ios_base::badbit;
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(ios_base::badbit);
}
return *this;
}
template<typename _CharT, typename _Traits>
basic_ostream<_CharT, _Traits>&
basic_ostream<_CharT, _Traits>::
flush()
{
if (__streambuf_type* __buf = this->rdbuf())
{
sentry __cerb(*this);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
if (this->rdbuf()->pubsync() == -1)
__err |= ios_base::badbit;
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(__err);
}
}
return *this;
}
template<typename _CharT, typename _Traits>
typename basic_ostream<_CharT, _Traits>::pos_type
basic_ostream<_CharT, _Traits>::
tellp()
{
sentry __cerb(*this);
pos_type __ret = pos_type(-1);
if (!this->fail())
__ret = this->rdbuf()->pubseekoff(0, ios_base::cur, ios_base::out);
return __ret;
}
template<typename _CharT, typename _Traits>
basic_ostream<_CharT, _Traits>&
basic_ostream<_CharT, _Traits>::
seekp(pos_type __pos)
{
sentry __cerb(*this);
if (!this->fail())
{
const pos_type __p = this->rdbuf()->pubseekpos(__pos, ios_base::out);
if (__p == pos_type(off_type(-1)))
this->setstate(ios_base::failbit);
}
return *this;
}
template<typename _CharT, typename _Traits>
basic_ostream<_CharT, _Traits>&
basic_ostream<_CharT, _Traits>::
seekp(off_type __off, ios_base::seekdir __dir)
{
sentry __cerb(*this);
if (!this->fail())
{
const pos_type __p = this->rdbuf()->pubseekoff(__off, __dir,
ios_base::out);
if (__p == pos_type(off_type(-1)))
this->setstate(ios_base::failbit);
}
return *this;
}
template<typename _CharT, typename _Traits>
basic_ostream<_CharT, _Traits>&
operator<<(basic_ostream<_CharT, _Traits>& __out, const char* __s)
{
if (!__s)
__out.setstate(ios_base::badbit);
else
{
const size_t __clen = char_traits<char>::length(__s);
try
{
struct __ptr_guard
{
_CharT *__p;
__ptr_guard (_CharT *__ip): __p(__ip) { }
~__ptr_guard() { delete[] __p; }
_CharT* __get() { return __p; }
} __pg (new _CharT[__clen]);
_CharT *__ws = __pg.__get();
for (size_t __i = 0; __i < __clen; ++__i)
__ws[__i] = __out.widen(__s[__i]);
__ostream_insert(__out, __ws, __clen);
}
catch(__cxxabiv1::__forced_unwind&)
{
__out._M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ __out._M_setstate(ios_base::badbit); }
}
return __out;
}
extern template class basic_ostream<char>;
extern template ostream& endl(ostream&);
extern template ostream& ends(ostream&);
extern template ostream& flush(ostream&);
extern template ostream& operator<<(ostream&, char);
extern template ostream& operator<<(ostream&, unsigned char);
extern template ostream& operator<<(ostream&, signed char);
extern template ostream& operator<<(ostream&, const char*);
extern template ostream& operator<<(ostream&, const unsigned char*);
extern template ostream& operator<<(ostream&, const signed char*);
extern template ostream& ostream::_M_insert(long);
extern template ostream& ostream::_M_insert(unsigned long);
extern template ostream& ostream::_M_insert(bool);
extern template ostream& ostream::_M_insert(long long);
extern template ostream& ostream::_M_insert(unsigned long long);
extern template ostream& ostream::_M_insert(double);
extern template ostream& ostream::_M_insert(long double);
extern template ostream& ostream::_M_insert(const void*);
extern template class basic_ostream<wchar_t>;
extern template wostream& endl(wostream&);
extern template wostream& ends(wostream&);
extern template wostream& flush(wostream&);
extern template wostream& operator<<(wostream&, wchar_t);
extern template wostream& operator<<(wostream&, char);
extern template wostream& operator<<(wostream&, const wchar_t*);
extern template wostream& operator<<(wostream&, const char*);
extern template wostream& wostream::_M_insert(long);
extern template wostream& wostream::_M_insert(unsigned long);
extern template wostream& wostream::_M_insert(bool);
extern template wostream& wostream::_M_insert(long long);
extern template wostream& wostream::_M_insert(unsigned long long);
extern template wostream& wostream::_M_insert(double);
extern template wostream& wostream::_M_insert(long double);
extern template wostream& wostream::_M_insert(const void*);
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _CharT, typename _Traits>
class basic_istream : virtual public basic_ios<_CharT, _Traits>
{
public:
typedef _CharT char_type;
typedef typename _Traits::int_type int_type;
typedef typename _Traits::pos_type pos_type;
typedef typename _Traits::off_type off_type;
typedef _Traits traits_type;
typedef basic_streambuf<_CharT, _Traits> __streambuf_type;
typedef basic_ios<_CharT, _Traits> __ios_type;
typedef basic_istream<_CharT, _Traits> __istream_type;
typedef num_get<_CharT, istreambuf_iterator<_CharT, _Traits> >
__num_get_type;
typedef ctype<_CharT> __ctype_type;
protected:
streamsize _M_gcount;
public:
explicit
basic_istream(__streambuf_type* __sb)
: _M_gcount(streamsize(0))
{ this->init(__sb); }
virtual
~basic_istream()
{ _M_gcount = streamsize(0); }
class sentry;
friend class sentry;
__istream_type&
operator>>(__istream_type& (*__pf)(__istream_type&))
{ return __pf(*this); }
__istream_type&
operator>>(__ios_type& (*__pf)(__ios_type&))
{
__pf(*this);
return *this;
}
__istream_type&
operator>>(ios_base& (*__pf)(ios_base&))
{
__pf(*this);
return *this;
}
__istream_type&
operator>>(bool& __n)
{ return _M_extract(__n); }
__istream_type&
operator>>(short& __n);
__istream_type&
operator>>(unsigned short& __n)
{ return _M_extract(__n); }
__istream_type&
operator>>(int& __n);
__istream_type&
operator>>(unsigned int& __n)
{ return _M_extract(__n); }
__istream_type&
operator>>(long& __n)
{ return _M_extract(__n); }
__istream_type&
operator>>(unsigned long& __n)
{ return _M_extract(__n); }
__istream_type&
operator>>(long long& __n)
{ return _M_extract(__n); }
__istream_type&
operator>>(unsigned long long& __n)
{ return _M_extract(__n); }
__istream_type&
operator>>(float& __f)
{ return _M_extract(__f); }
__istream_type&
operator>>(double& __f)
{ return _M_extract(__f); }
__istream_type&
operator>>(long double& __f)
{ return _M_extract(__f); }
__istream_type&
operator>>(void*& __p)
{ return _M_extract(__p); }
__istream_type&
operator>>(__streambuf_type* __sb);
streamsize
gcount() const
{ return _M_gcount; }
int_type
get();
__istream_type&
get(char_type& __c);
__istream_type&
get(char_type* __s, streamsize __n, char_type __delim);
__istream_type&
get(char_type* __s, streamsize __n)
{ return this->get(__s, __n, this->widen('\n')); }
__istream_type&
get(__streambuf_type& __sb, char_type __delim);
__istream_type&
get(__streambuf_type& __sb)
{ return this->get(__sb, this->widen('\n')); }
__istream_type&
getline(char_type* __s, streamsize __n, char_type __delim);
__istream_type&
getline(char_type* __s, streamsize __n)
{ return this->getline(__s, __n, this->widen('\n')); }
__istream_type&
ignore(streamsize __n, int_type __delim);
__istream_type&
ignore(streamsize __n);
__istream_type&
ignore();
int_type
peek();
__istream_type&
read(char_type* __s, streamsize __n);
streamsize
readsome(char_type* __s, streamsize __n);
__istream_type&
putback(char_type __c);
__istream_type&
unget();
int
sync();
pos_type
tellg();
__istream_type&
seekg(pos_type);
__istream_type&
seekg(off_type, ios_base::seekdir);
protected:
basic_istream()
: _M_gcount(streamsize(0))
{ this->init(0); }
basic_istream(const basic_istream&) = delete;
basic_istream(basic_istream&& __rhs)
: __ios_type(), _M_gcount(__rhs._M_gcount)
{
__ios_type::move(__rhs);
__rhs._M_gcount = 0;
}
basic_istream& operator=(const basic_istream&) = delete;
basic_istream&
operator=(basic_istream&& __rhs)
{
swap(__rhs);
return *this;
}
void
swap(basic_istream& __rhs)
{
__ios_type::swap(__rhs);
std::swap(_M_gcount, __rhs._M_gcount);
}
template<typename _ValueT>
__istream_type&
_M_extract(_ValueT& __v);
};
template<>
basic_istream<char>&
basic_istream<char>::
getline(char_type* __s, streamsize __n, char_type __delim);
template<>
basic_istream<char>&
basic_istream<char>::
ignore(streamsize __n);
template<>
basic_istream<char>&
basic_istream<char>::
ignore(streamsize __n, int_type __delim);
template<>
basic_istream<wchar_t>&
basic_istream<wchar_t>::
getline(char_type* __s, streamsize __n, char_type __delim);
template<>
basic_istream<wchar_t>&
basic_istream<wchar_t>::
ignore(streamsize __n);
template<>
basic_istream<wchar_t>&
basic_istream<wchar_t>::
ignore(streamsize __n, int_type __delim);
template<typename _CharT, typename _Traits>
class basic_istream<_CharT, _Traits>::sentry
{
bool _M_ok;
public:
typedef _Traits traits_type;
typedef basic_streambuf<_CharT, _Traits> __streambuf_type;
typedef basic_istream<_CharT, _Traits> __istream_type;
typedef typename __istream_type::__ctype_type __ctype_type;
typedef typename _Traits::int_type __int_type;
explicit
sentry(basic_istream<_CharT, _Traits>& __is, bool __noskipws = false);
explicit
operator bool() const
{ return _M_ok; }
};
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
operator>>(basic_istream<_CharT, _Traits>& __in, _CharT& __c);
template<class _Traits>
inline basic_istream<char, _Traits>&
operator>>(basic_istream<char, _Traits>& __in, unsigned char& __c)
{ return (__in >> reinterpret_cast<char&>(__c)); }
template<class _Traits>
inline basic_istream<char, _Traits>&
operator>>(basic_istream<char, _Traits>& __in, signed char& __c)
{ return (__in >> reinterpret_cast<char&>(__c)); }
template<typename _CharT, typename _Traits>
void
__istream_extract(basic_istream<_CharT, _Traits>&, _CharT*, streamsize);
void __istream_extract(istream&, char*, streamsize);
template<typename _CharT, typename _Traits>
__attribute__((__nonnull__(2), __access__(__write_only__, 2)))
inline basic_istream<_CharT, _Traits>&
operator>>(basic_istream<_CharT, _Traits>& __in, _CharT* __s)
{
size_t __n = __builtin_object_size(__s, 0);
if (__n < sizeof(_CharT))
{
do { if (std::__is_constant_evaluated() && !bool(__n >= sizeof(_CharT))) __builtin_unreachable(); } while (false);
__in.width(0);
__in.setstate(ios_base::failbit);
}
else if (__n != (size_t)-1)
{
__n /= sizeof(_CharT);
streamsize __w = __in.width();
std::__istream_extract(__in, __s, __n);
if (__in.good() && (__w <= 0 || __n < __w))
{
const typename _Traits::int_type __c = __in.rdbuf()->sgetc();
const bool __eof = _Traits::eq_int_type(__c, _Traits::eof());
if (__builtin_expect(__eof, true))
__in.setstate(ios_base::eofbit);
}
}
else
{
streamsize __n = __gnu_cxx::__numeric_traits<streamsize>::__max;
__n /= sizeof(_CharT);
std::__istream_extract(__in, __s, __n);
}
return __in;
}
template<class _Traits>
__attribute__((__nonnull__(2), __access__(__write_only__, 2)))
inline basic_istream<char, _Traits>&
operator>>(basic_istream<char, _Traits>& __in, unsigned char* __s)
{ return __in >> reinterpret_cast<char*>(__s); }
template<class _Traits>
__attribute__((__nonnull__(2), __access__(__write_only__, 2)))
inline basic_istream<char, _Traits>&
operator>>(basic_istream<char, _Traits>& __in, signed char* __s)
{ return __in >> reinterpret_cast<char*>(__s); }
template<typename _CharT, typename _Traits>
class basic_iostream
: public basic_istream<_CharT, _Traits>,
public basic_ostream<_CharT, _Traits>
{
public:
typedef _CharT char_type;
typedef typename _Traits::int_type int_type;
typedef typename _Traits::pos_type pos_type;
typedef typename _Traits::off_type off_type;
typedef _Traits traits_type;
typedef basic_istream<_CharT, _Traits> __istream_type;
typedef basic_ostream<_CharT, _Traits> __ostream_type;
explicit
basic_iostream(basic_streambuf<_CharT, _Traits>* __sb)
: __istream_type(__sb), __ostream_type(__sb) { }
virtual
~basic_iostream() { }
protected:
basic_iostream()
: __istream_type(), __ostream_type() { }
basic_iostream(const basic_iostream&) = delete;
basic_iostream(basic_iostream&& __rhs)
: __istream_type(std::move(__rhs)), __ostream_type(*this)
{ }
basic_iostream& operator=(const basic_iostream&) = delete;
basic_iostream&
operator=(basic_iostream&& __rhs)
{
swap(__rhs);
return *this;
}
void
swap(basic_iostream& __rhs)
{ __istream_type::swap(__rhs); }
};
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
ws(basic_istream<_CharT, _Traits>& __is);
template<typename _Is, typename _Tp,
typename = _Require_derived_from_ios_base<_Is>,
typename = decltype(std::declval<_Is&>() >> std::declval<_Tp>())>
using __rvalue_stream_extraction_t = _Is&&;
template<typename _Istream, typename _Tp>
inline __rvalue_stream_extraction_t<_Istream, _Tp>
operator>>(_Istream&& __is, _Tp&& __x)
{
__is >> std::forward<_Tp>(__x);
return std::move(__is);
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>::sentry::
sentry(basic_istream<_CharT, _Traits>& __in, bool __noskip) : _M_ok(false)
{
ios_base::iostate __err = ios_base::goodbit;
if (__in.good())
{
try
{
if (__in.tie())
__in.tie()->flush();
if (!__noskip && bool(__in.flags() & ios_base::skipws))
{
const __int_type __eof = traits_type::eof();
__streambuf_type* __sb = __in.rdbuf();
__int_type __c = __sb->sgetc();
const __ctype_type& __ct = __check_facet(__in._M_ctype);
while (!traits_type::eq_int_type(__c, __eof)
&& __ct.is(ctype_base::space,
traits_type::to_char_type(__c)))
__c = __sb->snextc();
if (traits_type::eq_int_type(__c, __eof))
__err |= ios_base::eofbit;
}
}
catch(__cxxabiv1::__forced_unwind&)
{
__in._M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ __in._M_setstate(ios_base::badbit); }
}
if (__in.good() && __err == ios_base::goodbit)
_M_ok = true;
else
{
__err |= ios_base::failbit;
__in.setstate(__err);
}
}
template<typename _CharT, typename _Traits>
template<typename _ValueT>
basic_istream<_CharT, _Traits>&
basic_istream<_CharT, _Traits>::
_M_extract(_ValueT& __v)
{
sentry __cerb(*this, false);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
const __num_get_type& __ng = __check_facet(this->_M_num_get);
__ng.get(*this, 0, *this, __err, __v);
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(__err);
}
return *this;
}
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
basic_istream<_CharT, _Traits>::
operator>>(short& __n)
{
sentry __cerb(*this, false);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
long __l;
const __num_get_type& __ng = __check_facet(this->_M_num_get);
__ng.get(*this, 0, *this, __err, __l);
if (__l < __gnu_cxx::__numeric_traits<short>::__min)
{
__err |= ios_base::failbit;
__n = __gnu_cxx::__numeric_traits<short>::__min;
}
else if (__l > __gnu_cxx::__numeric_traits<short>::__max)
{
__err |= ios_base::failbit;
__n = __gnu_cxx::__numeric_traits<short>::__max;
}
else
__n = short(__l);
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(__err);
}
return *this;
}
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
basic_istream<_CharT, _Traits>::
operator>>(int& __n)
{
sentry __cerb(*this, false);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
long __l;
const __num_get_type& __ng = __check_facet(this->_M_num_get);
__ng.get(*this, 0, *this, __err, __l);
if (__l < __gnu_cxx::__numeric_traits<int>::__min)
{
__err |= ios_base::failbit;
__n = __gnu_cxx::__numeric_traits<int>::__min;
}
else if (__l > __gnu_cxx::__numeric_traits<int>::__max)
{
__err |= ios_base::failbit;
__n = __gnu_cxx::__numeric_traits<int>::__max;
}
else
__n = int(__l);
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(__err);
}
return *this;
}
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
basic_istream<_CharT, _Traits>::
operator>>(__streambuf_type* __sbout)
{
ios_base::iostate __err = ios_base::goodbit;
sentry __cerb(*this, false);
if (__cerb && __sbout)
{
try
{
bool __ineof;
if (!__copy_streambufs_eof(this->rdbuf(), __sbout, __ineof))
__err |= ios_base::failbit;
if (__ineof)
__err |= ios_base::eofbit;
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::failbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::failbit); }
}
else if (!__sbout)
__err |= ios_base::failbit;
if (__err)
this->setstate(__err);
return *this;
}
template<typename _CharT, typename _Traits>
typename basic_istream<_CharT, _Traits>::int_type
basic_istream<_CharT, _Traits>::
get(void)
{
const int_type __eof = traits_type::eof();
int_type __c = __eof;
_M_gcount = 0;
ios_base::iostate __err = ios_base::goodbit;
sentry __cerb(*this, true);
if (__cerb)
{
try
{
__c = this->rdbuf()->sbumpc();
if (!traits_type::eq_int_type(__c, __eof))
_M_gcount = 1;
else
__err |= ios_base::eofbit;
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
}
if (!_M_gcount)
__err |= ios_base::failbit;
if (__err)
this->setstate(__err);
return __c;
}
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
basic_istream<_CharT, _Traits>::
get(char_type& __c)
{
_M_gcount = 0;
ios_base::iostate __err = ios_base::goodbit;
sentry __cerb(*this, true);
if (__cerb)
{
try
{
const int_type __cb = this->rdbuf()->sbumpc();
if (!traits_type::eq_int_type(__cb, traits_type::eof()))
{
_M_gcount = 1;
__c = traits_type::to_char_type(__cb);
}
else
__err |= ios_base::eofbit;
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
}
if (!_M_gcount)
__err |= ios_base::failbit;
if (__err)
this->setstate(__err);
return *this;
}
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
basic_istream<_CharT, _Traits>::
get(char_type* __s, streamsize __n, char_type __delim)
{
_M_gcount = 0;
ios_base::iostate __err = ios_base::goodbit;
sentry __cerb(*this, true);
if (__cerb)
{
try
{
const int_type __idelim = traits_type::to_int_type(__delim);
const int_type __eof = traits_type::eof();
__streambuf_type* __sb = this->rdbuf();
int_type __c = __sb->sgetc();
while (_M_gcount + 1 < __n
&& !traits_type::eq_int_type(__c, __eof)
&& !traits_type::eq_int_type(__c, __idelim))
{
*__s++ = traits_type::to_char_type(__c);
++_M_gcount;
__c = __sb->snextc();
}
if (traits_type::eq_int_type(__c, __eof))
__err |= ios_base::eofbit;
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
}
if (__n > 0)
*__s = char_type();
if (!_M_gcount)
__err |= ios_base::failbit;
if (__err)
this->setstate(__err);
return *this;
}
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
basic_istream<_CharT, _Traits>::
get(__streambuf_type& __sb, char_type __delim)
{
_M_gcount = 0;
ios_base::iostate __err = ios_base::goodbit;
sentry __cerb(*this, true);
if (__cerb)
{
try
{
const int_type __idelim = traits_type::to_int_type(__delim);
const int_type __eof = traits_type::eof();
__streambuf_type* __this_sb = this->rdbuf();
int_type __c = __this_sb->sgetc();
char_type __c2 = traits_type::to_char_type(__c);
unsigned long long __gcount = 0;
while (!traits_type::eq_int_type(__c, __eof)
&& !traits_type::eq_int_type(__c, __idelim)
&& !traits_type::eq_int_type(__sb.sputc(__c2), __eof))
{
++__gcount;
__c = __this_sb->snextc();
__c2 = traits_type::to_char_type(__c);
}
if (traits_type::eq_int_type(__c, __eof))
__err |= ios_base::eofbit;
if (__gcount <= __gnu_cxx::__numeric_traits<streamsize>::__max)
_M_gcount = __gcount;
else
_M_gcount = __gnu_cxx::__numeric_traits<streamsize>::__max;
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
}
if (!_M_gcount)
__err |= ios_base::failbit;
if (__err)
this->setstate(__err);
return *this;
}
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
basic_istream<_CharT, _Traits>::
getline(char_type* __s, streamsize __n, char_type __delim)
{
_M_gcount = 0;
ios_base::iostate __err = ios_base::goodbit;
sentry __cerb(*this, true);
if (__cerb)
{
try
{
const int_type __idelim = traits_type::to_int_type(__delim);
const int_type __eof = traits_type::eof();
__streambuf_type* __sb = this->rdbuf();
int_type __c = __sb->sgetc();
while (_M_gcount + 1 < __n
&& !traits_type::eq_int_type(__c, __eof)
&& !traits_type::eq_int_type(__c, __idelim))
{
*__s++ = traits_type::to_char_type(__c);
__c = __sb->snextc();
++_M_gcount;
}
if (traits_type::eq_int_type(__c, __eof))
__err |= ios_base::eofbit;
else
{
if (traits_type::eq_int_type(__c, __idelim))
{
__sb->sbumpc();
++_M_gcount;
}
else
__err |= ios_base::failbit;
}
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
}
if (__n > 0)
*__s = char_type();
if (!_M_gcount)
__err |= ios_base::failbit;
if (__err)
this->setstate(__err);
return *this;
}
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
basic_istream<_CharT, _Traits>::
ignore(void)
{
_M_gcount = 0;
sentry __cerb(*this, true);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
const int_type __eof = traits_type::eof();
__streambuf_type* __sb = this->rdbuf();
if (traits_type::eq_int_type(__sb->sbumpc(), __eof))
__err |= ios_base::eofbit;
else
_M_gcount = 1;
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(__err);
}
return *this;
}
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
basic_istream<_CharT, _Traits>::
ignore(streamsize __n)
{
_M_gcount = 0;
sentry __cerb(*this, true);
if (__cerb && __n > 0)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
const int_type __eof = traits_type::eof();
__streambuf_type* __sb = this->rdbuf();
int_type __c = __sb->sgetc();
bool __large_ignore = false;
while (true)
{
while (_M_gcount < __n
&& !traits_type::eq_int_type(__c, __eof))
{
++_M_gcount;
__c = __sb->snextc();
}
if (__n == __gnu_cxx::__numeric_traits<streamsize>::__max
&& !traits_type::eq_int_type(__c, __eof))
{
_M_gcount =
__gnu_cxx::__numeric_traits<streamsize>::__min;
__large_ignore = true;
}
else
break;
}
if (__n == __gnu_cxx::__numeric_traits<streamsize>::__max)
{
if (__large_ignore)
_M_gcount = __gnu_cxx::__numeric_traits<streamsize>::__max;
if (traits_type::eq_int_type(__c, __eof))
__err |= ios_base::eofbit;
}
else if (_M_gcount < __n)
{
if (traits_type::eq_int_type(__c, __eof))
__err |= ios_base::eofbit;
}
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(__err);
}
return *this;
}
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
basic_istream<_CharT, _Traits>::
ignore(streamsize __n, int_type __delim)
{
_M_gcount = 0;
sentry __cerb(*this, true);
if (__cerb && __n > 0)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
const int_type __eof = traits_type::eof();
__streambuf_type* __sb = this->rdbuf();
int_type __c = __sb->sgetc();
bool __large_ignore = false;
while (true)
{
while (_M_gcount < __n
&& !traits_type::eq_int_type(__c, __eof)
&& !traits_type::eq_int_type(__c, __delim))
{
++_M_gcount;
__c = __sb->snextc();
}
if (__n == __gnu_cxx::__numeric_traits<streamsize>::__max
&& !traits_type::eq_int_type(__c, __eof)
&& !traits_type::eq_int_type(__c, __delim))
{
_M_gcount =
__gnu_cxx::__numeric_traits<streamsize>::__min;
__large_ignore = true;
}
else
break;
}
if (__n == __gnu_cxx::__numeric_traits<streamsize>::__max)
{
if (__large_ignore)
_M_gcount = __gnu_cxx::__numeric_traits<streamsize>::__max;
if (traits_type::eq_int_type(__c, __eof))
__err |= ios_base::eofbit;
else
{
if (_M_gcount != __n)
++_M_gcount;
__sb->sbumpc();
}
}
else if (_M_gcount < __n)
{
if (traits_type::eq_int_type(__c, __eof))
__err |= ios_base::eofbit;
else
{
++_M_gcount;
__sb->sbumpc();
}
}
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(__err);
}
return *this;
}
template<typename _CharT, typename _Traits>
typename basic_istream<_CharT, _Traits>::int_type
basic_istream<_CharT, _Traits>::
peek(void)
{
int_type __c = traits_type::eof();
_M_gcount = 0;
sentry __cerb(*this, true);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
__c = this->rdbuf()->sgetc();
if (traits_type::eq_int_type(__c, traits_type::eof()))
__err |= ios_base::eofbit;
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(__err);
}
return __c;
}
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
basic_istream<_CharT, _Traits>::
read(char_type* __s, streamsize __n)
{
_M_gcount = 0;
sentry __cerb(*this, true);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
_M_gcount = this->rdbuf()->sgetn(__s, __n);
if (_M_gcount != __n)
__err |= (ios_base::eofbit | ios_base::failbit);
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(__err);
}
return *this;
}
template<typename _CharT, typename _Traits>
streamsize
basic_istream<_CharT, _Traits>::
readsome(char_type* __s, streamsize __n)
{
_M_gcount = 0;
sentry __cerb(*this, true);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
const streamsize __num = this->rdbuf()->in_avail();
if (__num > 0)
_M_gcount = this->rdbuf()->sgetn(__s, std::min(__num, __n));
else if (__num == -1)
__err |= ios_base::eofbit;
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(__err);
}
return _M_gcount;
}
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
basic_istream<_CharT, _Traits>::
putback(char_type __c)
{
_M_gcount = 0;
this->clear(this->rdstate() & ~ios_base::eofbit);
sentry __cerb(*this, true);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
const int_type __eof = traits_type::eof();
__streambuf_type* __sb = this->rdbuf();
if (!__sb
|| traits_type::eq_int_type(__sb->sputbackc(__c), __eof))
__err |= ios_base::badbit;
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(__err);
}
return *this;
}
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
basic_istream<_CharT, _Traits>::
unget(void)
{
_M_gcount = 0;
this->clear(this->rdstate() & ~ios_base::eofbit);
sentry __cerb(*this, true);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
const int_type __eof = traits_type::eof();
__streambuf_type* __sb = this->rdbuf();
if (!__sb
|| traits_type::eq_int_type(__sb->sungetc(), __eof))
__err |= ios_base::badbit;
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(__err);
}
return *this;
}
template<typename _CharT, typename _Traits>
int
basic_istream<_CharT, _Traits>::
sync(void)
{
int __ret = -1;
sentry __cerb(*this, true);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
__streambuf_type* __sb = this->rdbuf();
if (__sb)
{
if (__sb->pubsync() == -1)
__err |= ios_base::badbit;
else
__ret = 0;
}
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(__err);
}
return __ret;
}
template<typename _CharT, typename _Traits>
typename basic_istream<_CharT, _Traits>::pos_type
basic_istream<_CharT, _Traits>::
tellg(void)
{
pos_type __ret = pos_type(-1);
sentry __cerb(*this, true);
if (__cerb)
{
try
{
if (!this->fail())
__ret = this->rdbuf()->pubseekoff(0, ios_base::cur,
ios_base::in);
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
}
return __ret;
}
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
basic_istream<_CharT, _Traits>::
seekg(pos_type __pos)
{
this->clear(this->rdstate() & ~ios_base::eofbit);
sentry __cerb(*this, true);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
if (!this->fail())
{
const pos_type __p = this->rdbuf()->pubseekpos(__pos,
ios_base::in);
if (__p == pos_type(off_type(-1)))
__err |= ios_base::failbit;
}
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(__err);
}
return *this;
}
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
basic_istream<_CharT, _Traits>::
seekg(off_type __off, ios_base::seekdir __dir)
{
this->clear(this->rdstate() & ~ios_base::eofbit);
sentry __cerb(*this, true);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
if (!this->fail())
{
const pos_type __p = this->rdbuf()->pubseekoff(__off, __dir,
ios_base::in);
if (__p == pos_type(off_type(-1)))
__err |= ios_base::failbit;
}
}
catch(__cxxabiv1::__forced_unwind&)
{
this->_M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ this->_M_setstate(ios_base::badbit); }
if (__err)
this->setstate(__err);
}
return *this;
}
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
operator>>(basic_istream<_CharT, _Traits>& __in, _CharT& __c)
{
typedef basic_istream<_CharT, _Traits> __istream_type;
typedef typename __istream_type::int_type __int_type;
typename __istream_type::sentry __cerb(__in, false);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
const __int_type __cb = __in.rdbuf()->sbumpc();
if (!_Traits::eq_int_type(__cb, _Traits::eof()))
__c = _Traits::to_char_type(__cb);
else
__err |= (ios_base::eofbit | ios_base::failbit);
}
catch(__cxxabiv1::__forced_unwind&)
{
__in._M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ __in._M_setstate(ios_base::badbit); }
if (__err)
__in.setstate(__err);
}
return __in;
}
template<typename _CharT, typename _Traits>
void
__istream_extract(basic_istream<_CharT, _Traits>& __in, _CharT* __s,
streamsize __num)
{
typedef basic_istream<_CharT, _Traits> __istream_type;
typedef basic_streambuf<_CharT, _Traits> __streambuf_type;
typedef typename _Traits::int_type int_type;
typedef _CharT char_type;
typedef ctype<_CharT> __ctype_type;
streamsize __extracted = 0;
ios_base::iostate __err = ios_base::goodbit;
typename __istream_type::sentry __cerb(__in, false);
if (__cerb)
{
try
{
streamsize __width = __in.width();
if (0 < __width && __width < __num)
__num = __width;
const __ctype_type& __ct = use_facet<__ctype_type>(__in.getloc());
const int_type __eof = _Traits::eof();
__streambuf_type* __sb = __in.rdbuf();
int_type __c = __sb->sgetc();
while (__extracted < __num - 1
&& !_Traits::eq_int_type(__c, __eof)
&& !__ct.is(ctype_base::space,
_Traits::to_char_type(__c)))
{
*__s++ = _Traits::to_char_type(__c);
++__extracted;
__c = __sb->snextc();
}
if (__extracted < __num - 1
&& _Traits::eq_int_type(__c, __eof))
__err |= ios_base::eofbit;
*__s = char_type();
__in.width(0);
}
catch(__cxxabiv1::__forced_unwind&)
{
__in._M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ __in._M_setstate(ios_base::badbit); }
}
if (!__extracted)
__err |= ios_base::failbit;
if (__err)
__in.setstate(__err);
}
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
ws(basic_istream<_CharT, _Traits>& __in)
{
typedef basic_istream<_CharT, _Traits> __istream_type;
typedef basic_streambuf<_CharT, _Traits> __streambuf_type;
typedef typename __istream_type::int_type __int_type;
typedef ctype<_CharT> __ctype_type;
typename __istream_type::sentry __cerb(__in, true);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
const __ctype_type& __ct = use_facet<__ctype_type>(__in.getloc());
const __int_type __eof = _Traits::eof();
__streambuf_type* __sb = __in.rdbuf();
__int_type __c = __sb->sgetc();
while (true)
{
if (_Traits::eq_int_type(__c, __eof))
{
__err = ios_base::eofbit;
break;
}
if (!__ct.is(ctype_base::space, _Traits::to_char_type(__c)))
break;
__c = __sb->snextc();
}
}
catch(const __cxxabiv1::__forced_unwind&)
{
__in._M_setstate(ios_base::badbit);
throw;
}
catch(...)
{
__in._M_setstate(ios_base::badbit);
}
if (__err)
__in.setstate(__err);
}
return __in;
}
extern template class basic_istream<char>;
extern template istream& ws(istream&);
extern template istream& operator>>(istream&, char&);
extern template istream& operator>>(istream&, unsigned char&);
extern template istream& operator>>(istream&, signed char&);
extern template istream& istream::_M_extract(unsigned short&);
extern template istream& istream::_M_extract(unsigned int&);
extern template istream& istream::_M_extract(long&);
extern template istream& istream::_M_extract(unsigned long&);
extern template istream& istream::_M_extract(bool&);
extern template istream& istream::_M_extract(long long&);
extern template istream& istream::_M_extract(unsigned long long&);
extern template istream& istream::_M_extract(float&);
extern template istream& istream::_M_extract(double&);
extern template istream& istream::_M_extract(long double&);
extern template istream& istream::_M_extract(void*&);
extern template class basic_iostream<char>;
extern template class basic_istream<wchar_t>;
extern template wistream& ws(wistream&);
extern template wistream& operator>>(wistream&, wchar_t&);
extern template void __istream_extract(wistream&, wchar_t*, streamsize);
extern template wistream& wistream::_M_extract(unsigned short&);
extern template wistream& wistream::_M_extract(unsigned int&);
extern template wistream& wistream::_M_extract(long&);
extern template wistream& wistream::_M_extract(unsigned long&);
extern template wistream& wistream::_M_extract(bool&);
extern template wistream& wistream::_M_extract(long long&);
extern template wistream& wistream::_M_extract(unsigned long long&);
extern template wistream& wistream::_M_extract(float&);
extern template wistream& wistream::_M_extract(double&);
extern template wistream& wistream::_M_extract(long double&);
extern template wistream& wistream::_M_extract(void*&);
extern template class basic_iostream<wchar_t>;
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __cxx11 {
template<typename _CharT, typename _Traits, typename _Alloc>
class basic_stringbuf : public basic_streambuf<_CharT, _Traits>
{
struct __xfer_bufptrs;
using allocator_traits = std::allocator_traits<_Alloc>;
using _Noexcept_swap
= __or_<typename allocator_traits::propagate_on_container_swap,
typename allocator_traits::is_always_equal>;
public:
typedef _CharT char_type;
typedef _Traits traits_type;
typedef _Alloc allocator_type;
typedef typename traits_type::int_type int_type;
typedef typename traits_type::pos_type pos_type;
typedef typename traits_type::off_type off_type;
typedef basic_streambuf<char_type, traits_type> __streambuf_type;
typedef basic_string<char_type, _Traits, _Alloc> __string_type;
typedef typename __string_type::size_type __size_type;
protected:
ios_base::openmode _M_mode;
__string_type _M_string;
public:
basic_stringbuf()
: __streambuf_type(), _M_mode(ios_base::in | ios_base::out), _M_string()
{ }
explicit
basic_stringbuf(ios_base::openmode __mode)
: __streambuf_type(), _M_mode(__mode), _M_string()
{ }
explicit
basic_stringbuf(const __string_type& __str,
ios_base::openmode __mode = ios_base::in | ios_base::out)
: __streambuf_type(), _M_mode(),
_M_string(__str.data(), __str.size(), __str.get_allocator())
{ _M_stringbuf_init(__mode); }
basic_stringbuf(const basic_stringbuf&) = delete;
basic_stringbuf(basic_stringbuf&& __rhs)
: basic_stringbuf(std::move(__rhs), __xfer_bufptrs(__rhs, this))
{ __rhs._M_sync(const_cast<char_type*>(__rhs._M_string.data()), 0, 0); }
basic_stringbuf&
operator=(const basic_stringbuf&) = delete;
basic_stringbuf&
operator=(basic_stringbuf&& __rhs)
{
__xfer_bufptrs __st{__rhs, this};
const __streambuf_type& __base = __rhs;
__streambuf_type::operator=(__base);
this->pubimbue(__rhs.getloc());
_M_mode = __rhs._M_mode;
_M_string = std::move(__rhs._M_string);
__rhs._M_sync(const_cast<char_type*>(__rhs._M_string.data()), 0, 0);
return *this;
}
void
swap(basic_stringbuf& __rhs) noexcept(_Noexcept_swap::value)
{
__xfer_bufptrs __l_st{*this, std::__addressof(__rhs)};
__xfer_bufptrs __r_st{__rhs, this};
__streambuf_type& __base = __rhs;
__streambuf_type::swap(__base);
__rhs.pubimbue(this->pubimbue(__rhs.getloc()));
std::swap(_M_mode, __rhs._M_mode);
std::swap(_M_string, __rhs._M_string);
}
__string_type
str() const
{
__string_type __ret(_M_string.get_allocator());
if (char_type* __hi = _M_high_mark())
__ret.assign(this->pbase(), __hi);
else
__ret = _M_string;
return __ret;
}
void
str(const __string_type& __s)
{
_M_string.assign(__s.data(), __s.size());
_M_stringbuf_init(_M_mode);
}
protected:
void
_M_stringbuf_init(ios_base::openmode __mode)
{
_M_mode = __mode;
__size_type __len = 0;
if (_M_mode & (ios_base::ate | ios_base::app))
__len = _M_string.size();
_M_sync(const_cast<char_type*>(_M_string.data()), 0, __len);
}
virtual streamsize
showmanyc()
{
streamsize __ret = -1;
if (_M_mode & ios_base::in)
{
_M_update_egptr();
__ret = this->egptr() - this->gptr();
}
return __ret;
}
virtual int_type
underflow();
virtual int_type
pbackfail(int_type __c = traits_type::eof());
virtual int_type
overflow(int_type __c = traits_type::eof());
virtual __streambuf_type*
setbuf(char_type* __s, streamsize __n)
{
if (__s && __n >= 0)
{
_M_string.clear();
_M_sync(__s, __n, 0);
}
return this;
}
virtual pos_type
seekoff(off_type __off, ios_base::seekdir __way,
ios_base::openmode __mode = ios_base::in | ios_base::out);
virtual pos_type
seekpos(pos_type __sp,
ios_base::openmode __mode = ios_base::in | ios_base::out);
void
_M_sync(char_type* __base, __size_type __i, __size_type __o);
void
_M_update_egptr()
{
if (char_type* __pptr = this->pptr())
{
char_type* __egptr = this->egptr();
if (!__egptr || __pptr > __egptr)
{
if (_M_mode & ios_base::in)
this->setg(this->eback(), this->gptr(), __pptr);
else
this->setg(__pptr, __pptr, __pptr);
}
}
}
void
_M_pbump(char_type* __pbeg, char_type* __pend, off_type __off);
private:
__attribute__((__always_inline__))
char_type*
_M_high_mark() const noexcept
{
if (char_type* __pptr = this->pptr())
{
char_type* __egptr = this->egptr();
if (!__egptr || __pptr > __egptr)
return __pptr;
else
return __egptr;
}
return 0;
}
struct __xfer_bufptrs
{
__xfer_bufptrs(const basic_stringbuf& __from, basic_stringbuf* __to)
: _M_to{__to}, _M_goff{-1, -1, -1}, _M_poff{-1, -1, -1}
{
const _CharT* const __str = __from._M_string.data();
const _CharT* __end = nullptr;
if (__from.eback())
{
_M_goff[0] = __from.eback() - __str;
_M_goff[1] = __from.gptr() - __str;
_M_goff[2] = __from.egptr() - __str;
__end = __from.egptr();
}
if (__from.pbase())
{
_M_poff[0] = __from.pbase() - __str;
_M_poff[1] = __from.pptr() - __from.pbase();
_M_poff[2] = __from.epptr() - __str;
if (!__end || __from.pptr() > __end)
__end = __from.pptr();
}
if (__end)
{
auto& __mut_from = const_cast<basic_stringbuf&>(__from);
__mut_from._M_string._M_length(__end - __str);
}
}
~__xfer_bufptrs()
{
char_type* __str = const_cast<char_type*>(_M_to->_M_string.data());
if (_M_goff[0] != -1)
_M_to->setg(__str+_M_goff[0], __str+_M_goff[1], __str+_M_goff[2]);
if (_M_poff[0] != -1)
_M_to->_M_pbump(__str+_M_poff[0], __str+_M_poff[2], _M_poff[1]);
}
basic_stringbuf* _M_to;
off_type _M_goff[3];
off_type _M_poff[3];
};
basic_stringbuf(basic_stringbuf&& __rhs, __xfer_bufptrs&&)
: __streambuf_type(static_cast<const __streambuf_type&>(__rhs)),
_M_mode(__rhs._M_mode), _M_string(std::move(__rhs._M_string))
{ }
};
template<typename _CharT, typename _Traits, typename _Alloc>
class basic_istringstream : public basic_istream<_CharT, _Traits>
{
public:
typedef _CharT char_type;
typedef _Traits traits_type;
typedef _Alloc allocator_type;
typedef typename traits_type::int_type int_type;
typedef typename traits_type::pos_type pos_type;
typedef typename traits_type::off_type off_type;
typedef basic_string<_CharT, _Traits, _Alloc> __string_type;
typedef basic_stringbuf<_CharT, _Traits, _Alloc> __stringbuf_type;
typedef basic_istream<char_type, traits_type> __istream_type;
private:
__stringbuf_type _M_stringbuf;
public:
basic_istringstream()
: __istream_type(), _M_stringbuf(ios_base::in)
{ this->init(&_M_stringbuf); }
explicit
basic_istringstream(ios_base::openmode __mode)
: __istream_type(), _M_stringbuf(__mode | ios_base::in)
{ this->init(&_M_stringbuf); }
explicit
basic_istringstream(const __string_type& __str,
ios_base::openmode __mode = ios_base::in)
: __istream_type(), _M_stringbuf(__str, __mode | ios_base::in)
{ this->init(&_M_stringbuf); }
~basic_istringstream()
{ }
basic_istringstream(const basic_istringstream&) = delete;
basic_istringstream(basic_istringstream&& __rhs)
: __istream_type(std::move(__rhs)),
_M_stringbuf(std::move(__rhs._M_stringbuf))
{ __istream_type::set_rdbuf(&_M_stringbuf); }
basic_istringstream&
operator=(const basic_istringstream&) = delete;
basic_istringstream&
operator=(basic_istringstream&& __rhs)
{
__istream_type::operator=(std::move(__rhs));
_M_stringbuf = std::move(__rhs._M_stringbuf);
return *this;
}
void
swap(basic_istringstream& __rhs)
{
__istream_type::swap(__rhs);
_M_stringbuf.swap(__rhs._M_stringbuf);
}
__stringbuf_type*
rdbuf() const
{ return const_cast<__stringbuf_type*>(&_M_stringbuf); }
__string_type
str() const
{ return _M_stringbuf.str(); }
void
str(const __string_type& __s)
{ _M_stringbuf.str(__s); }
};
template <typename _CharT, typename _Traits, typename _Alloc>
class basic_ostringstream : public basic_ostream<_CharT, _Traits>
{
public:
typedef _CharT char_type;
typedef _Traits traits_type;
typedef _Alloc allocator_type;
typedef typename traits_type::int_type int_type;
typedef typename traits_type::pos_type pos_type;
typedef typename traits_type::off_type off_type;
typedef basic_string<_CharT, _Traits, _Alloc> __string_type;
typedef basic_stringbuf<_CharT, _Traits, _Alloc> __stringbuf_type;
typedef basic_ostream<char_type, traits_type> __ostream_type;
private:
__stringbuf_type _M_stringbuf;
public:
basic_ostringstream()
: __ostream_type(), _M_stringbuf(ios_base::out)
{ this->init(&_M_stringbuf); }
explicit
basic_ostringstream(ios_base::openmode __mode)
: __ostream_type(), _M_stringbuf(__mode | ios_base::out)
{ this->init(&_M_stringbuf); }
explicit
basic_ostringstream(const __string_type& __str,
ios_base::openmode __mode = ios_base::out)
: __ostream_type(), _M_stringbuf(__str, __mode | ios_base::out)
{ this->init(&_M_stringbuf); }
~basic_ostringstream()
{ }
basic_ostringstream(const basic_ostringstream&) = delete;
basic_ostringstream(basic_ostringstream&& __rhs)
: __ostream_type(std::move(__rhs)),
_M_stringbuf(std::move(__rhs._M_stringbuf))
{ __ostream_type::set_rdbuf(&_M_stringbuf); }
basic_ostringstream&
operator=(const basic_ostringstream&) = delete;
basic_ostringstream&
operator=(basic_ostringstream&& __rhs)
{
__ostream_type::operator=(std::move(__rhs));
_M_stringbuf = std::move(__rhs._M_stringbuf);
return *this;
}
void
swap(basic_ostringstream& __rhs)
{
__ostream_type::swap(__rhs);
_M_stringbuf.swap(__rhs._M_stringbuf);
}
__stringbuf_type*
rdbuf() const
{ return const_cast<__stringbuf_type*>(&_M_stringbuf); }
__string_type
str() const
{ return _M_stringbuf.str(); }
void
str(const __string_type& __s)
{ _M_stringbuf.str(__s); }
};
template <typename _CharT, typename _Traits, typename _Alloc>
class basic_stringstream : public basic_iostream<_CharT, _Traits>
{
public:
typedef _CharT char_type;
typedef _Traits traits_type;
typedef _Alloc allocator_type;
typedef typename traits_type::int_type int_type;
typedef typename traits_type::pos_type pos_type;
typedef typename traits_type::off_type off_type;
typedef basic_string<_CharT, _Traits, _Alloc> __string_type;
typedef basic_stringbuf<_CharT, _Traits, _Alloc> __stringbuf_type;
typedef basic_iostream<char_type, traits_type> __iostream_type;
private:
__stringbuf_type _M_stringbuf;
public:
basic_stringstream()
: __iostream_type(), _M_stringbuf(ios_base::out | ios_base::in)
{ this->init(&_M_stringbuf); }
explicit
basic_stringstream(ios_base::openmode __m)
: __iostream_type(), _M_stringbuf(__m)
{ this->init(&_M_stringbuf); }
explicit
basic_stringstream(const __string_type& __str,
ios_base::openmode __m = ios_base::out | ios_base::in)
: __iostream_type(), _M_stringbuf(__str, __m)
{ this->init(&_M_stringbuf); }
~basic_stringstream()
{ }
basic_stringstream(const basic_stringstream&) = delete;
basic_stringstream(basic_stringstream&& __rhs)
: __iostream_type(std::move(__rhs)),
_M_stringbuf(std::move(__rhs._M_stringbuf))
{ __iostream_type::set_rdbuf(&_M_stringbuf); }
basic_stringstream&
operator=(const basic_stringstream&) = delete;
basic_stringstream&
operator=(basic_stringstream&& __rhs)
{
__iostream_type::operator=(std::move(__rhs));
_M_stringbuf = std::move(__rhs._M_stringbuf);
return *this;
}
void
swap(basic_stringstream& __rhs)
{
__iostream_type::swap(__rhs);
_M_stringbuf.swap(__rhs._M_stringbuf);
}
__stringbuf_type*
rdbuf() const
{ return const_cast<__stringbuf_type*>(&_M_stringbuf); }
__string_type
str() const
{ return _M_stringbuf.str(); }
void
str(const __string_type& __s)
{ _M_stringbuf.str(__s); }
};
template <class _CharT, class _Traits, class _Allocator>
inline void
swap(basic_stringbuf<_CharT, _Traits, _Allocator>& __x,
basic_stringbuf<_CharT, _Traits, _Allocator>& __y)
noexcept(noexcept(__x.swap(__y)))
{ __x.swap(__y); }
template <class _CharT, class _Traits, class _Allocator>
inline void
swap(basic_istringstream<_CharT, _Traits, _Allocator>& __x,
basic_istringstream<_CharT, _Traits, _Allocator>& __y)
{ __x.swap(__y); }
template <class _CharT, class _Traits, class _Allocator>
inline void
swap(basic_ostringstream<_CharT, _Traits, _Allocator>& __x,
basic_ostringstream<_CharT, _Traits, _Allocator>& __y)
{ __x.swap(__y); }
template <class _CharT, class _Traits, class _Allocator>
inline void
swap(basic_stringstream<_CharT, _Traits, _Allocator>& __x,
basic_stringstream<_CharT, _Traits, _Allocator>& __y)
{ __x.swap(__y); }
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template <class _CharT, class _Traits, class _Alloc>
typename basic_stringbuf<_CharT, _Traits, _Alloc>::int_type
basic_stringbuf<_CharT, _Traits, _Alloc>::
pbackfail(int_type __c)
{
int_type __ret = traits_type::eof();
if (this->eback() < this->gptr())
{
const bool __testeof = traits_type::eq_int_type(__c, __ret);
if (!__testeof)
{
const bool __testeq = traits_type::eq(traits_type::
to_char_type(__c),
this->gptr()[-1]);
const bool __testout = this->_M_mode & ios_base::out;
if (__testeq || __testout)
{
this->gbump(-1);
if (!__testeq)
*this->gptr() = traits_type::to_char_type(__c);
__ret = __c;
}
}
else
{
this->gbump(-1);
__ret = traits_type::not_eof(__c);
}
}
return __ret;
}
template <class _CharT, class _Traits, class _Alloc>
typename basic_stringbuf<_CharT, _Traits, _Alloc>::int_type
basic_stringbuf<_CharT, _Traits, _Alloc>::
overflow(int_type __c)
{
const bool __testout = this->_M_mode & ios_base::out;
if (__builtin_expect(!__testout, false))
return traits_type::eof();
const bool __testeof = traits_type::eq_int_type(__c, traits_type::eof());
if (__builtin_expect(__testeof, false))
return traits_type::not_eof(__c);
const __size_type __capacity = _M_string.capacity();
if (size_t(this->epptr() - this->pbase()) < __capacity)
{
char_type* __base = const_cast<char_type*>(_M_string.data());
_M_pbump(__base, __base + __capacity, this->pptr() - this->pbase());
if (_M_mode & ios_base::in)
{
const __size_type __nget = this->gptr() - this->eback();
const __size_type __eget = this->egptr() - this->eback();
this->setg(__base, __base + __nget, __base + __eget + 1);
}
*this->pptr() = traits_type::to_char_type(__c);
this->pbump(1);
return __c;
}
const __size_type __max_size = _M_string.max_size();
const bool __testput = this->pptr() < this->epptr();
if (__builtin_expect(!__testput && __capacity == __max_size, false))
return traits_type::eof();
const char_type __conv = traits_type::to_char_type(__c);
if (!__testput)
{
const __size_type __opt_len = std::max(__size_type(2 * __capacity),
__size_type(512));
const __size_type __len = std::min(__opt_len, __max_size);
__string_type __tmp(_M_string.get_allocator());
__tmp.reserve(__len);
if (this->pbase())
__tmp.assign(this->pbase(), this->epptr() - this->pbase());
__tmp.push_back(__conv);
_M_string.swap(__tmp);
_M_sync(const_cast<char_type*>(_M_string.data()),
this->gptr() - this->eback(), this->pptr() - this->pbase());
}
else
*this->pptr() = __conv;
this->pbump(1);
return __c;
}
template <class _CharT, class _Traits, class _Alloc>
typename basic_stringbuf<_CharT, _Traits, _Alloc>::int_type
basic_stringbuf<_CharT, _Traits, _Alloc>::
underflow()
{
int_type __ret = traits_type::eof();
const bool __testin = this->_M_mode & ios_base::in;
if (__testin)
{
_M_update_egptr();
if (this->gptr() < this->egptr())
__ret = traits_type::to_int_type(*this->gptr());
}
return __ret;
}
template <class _CharT, class _Traits, class _Alloc>
typename basic_stringbuf<_CharT, _Traits, _Alloc>::pos_type
basic_stringbuf<_CharT, _Traits, _Alloc>::
seekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __mode)
{
pos_type __ret = pos_type(off_type(-1));
bool __testin = (ios_base::in & this->_M_mode & __mode) != 0;
bool __testout = (ios_base::out & this->_M_mode & __mode) != 0;
const bool __testboth = __testin && __testout && __way != ios_base::cur;
__testin &= !(__mode & ios_base::out);
__testout &= !(__mode & ios_base::in);
const char_type* __beg = __testin ? this->eback() : this->pbase();
if ((__beg || !__off) && (__testin || __testout || __testboth))
{
_M_update_egptr();
off_type __newoffi = __off;
off_type __newoffo = __newoffi;
if (__way == ios_base::cur)
{
__newoffi += this->gptr() - __beg;
__newoffo += this->pptr() - __beg;
}
else if (__way == ios_base::end)
__newoffo = __newoffi += this->egptr() - __beg;
if ((__testin || __testboth)
&& __newoffi >= 0
&& this->egptr() - __beg >= __newoffi)
{
this->setg(this->eback(), this->eback() + __newoffi,
this->egptr());
__ret = pos_type(__newoffi);
}
if ((__testout || __testboth)
&& __newoffo >= 0
&& this->egptr() - __beg >= __newoffo)
{
_M_pbump(this->pbase(), this->epptr(), __newoffo);
__ret = pos_type(__newoffo);
}
}
return __ret;
}
template <class _CharT, class _Traits, class _Alloc>
typename basic_stringbuf<_CharT, _Traits, _Alloc>::pos_type
basic_stringbuf<_CharT, _Traits, _Alloc>::
seekpos(pos_type __sp, ios_base::openmode __mode)
{
pos_type __ret = pos_type(off_type(-1));
const bool __testin = (ios_base::in & this->_M_mode & __mode) != 0;
const bool __testout = (ios_base::out & this->_M_mode & __mode) != 0;
const char_type* __beg = __testin ? this->eback() : this->pbase();
if ((__beg || !off_type(__sp)) && (__testin || __testout))
{
_M_update_egptr();
const off_type __pos(__sp);
const bool __testpos = (0 <= __pos
&& __pos <= this->egptr() - __beg);
if (__testpos)
{
if (__testin)
this->setg(this->eback(), this->eback() + __pos,
this->egptr());
if (__testout)
_M_pbump(this->pbase(), this->epptr(), __pos);
__ret = __sp;
}
}
return __ret;
}
template <class _CharT, class _Traits, class _Alloc>
void
basic_stringbuf<_CharT, _Traits, _Alloc>::
_M_sync(char_type* __base, __size_type __i, __size_type __o)
{
const bool __testin = _M_mode & ios_base::in;
const bool __testout = _M_mode & ios_base::out;
char_type* __endg = __base + _M_string.size();
char_type* __endp = __base + _M_string.capacity();
if (__base != _M_string.data())
{
__endg += __i;
__i = 0;
__endp = __endg;
}
if (__testin)
this->setg(__base, __base + __i, __endg);
if (__testout)
{
_M_pbump(__base, __endp, __o);
if (!__testin)
this->setg(__endg, __endg, __endg);
}
}
template <class _CharT, class _Traits, class _Alloc>
void
basic_stringbuf<_CharT, _Traits, _Alloc>::
_M_pbump(char_type* __pbeg, char_type* __pend, off_type __off)
{
this->setp(__pbeg, __pend);
while (__off > __gnu_cxx::__numeric_traits<int>::__max)
{
this->pbump(__gnu_cxx::__numeric_traits<int>::__max);
__off -= __gnu_cxx::__numeric_traits<int>::__max;
}
this->pbump(__off);
}
extern template class basic_stringbuf<char>;
extern template class basic_istringstream<char>;
extern template class basic_ostringstream<char>;
extern template class basic_stringstream<char>;
extern template class basic_stringbuf<wchar_t>;
extern template class basic_istringstream<wchar_t>;
extern template class basic_ostringstream<wchar_t>;
extern template class basic_stringstream<wchar_t>;
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __detail {
template<typename _String, typename _CharT>
struct _Quoted_string
{
static_assert(is_reference<_String>::value
|| is_pointer<_String>::value,
"String type must be pointer or reference");
_Quoted_string(_String __str, _CharT __del, _CharT __esc)
: _M_string(__str), _M_delim{__del}, _M_escape{__esc}
{ }
_Quoted_string&
operator=(_Quoted_string&) = delete;
_String _M_string;
_CharT _M_delim;
_CharT _M_escape;
};
template<typename _CharT, typename _Traits>
struct _Quoted_string<basic_string_view<_CharT, _Traits>, _CharT>
{
_Quoted_string(basic_string_view<_CharT, _Traits> __str,
_CharT __del, _CharT __esc)
: _M_string(__str), _M_delim{__del}, _M_escape{__esc}
{ }
_Quoted_string&
operator=(_Quoted_string&) = delete;
basic_string_view<_CharT, _Traits> _M_string;
_CharT _M_delim;
_CharT _M_escape;
};
template<typename _CharT, typename _Traits>
std::basic_ostream<_CharT, _Traits>&
operator<<(std::basic_ostream<_CharT, _Traits>& __os,
const _Quoted_string<const _CharT*, _CharT>& __str)
{
std::basic_ostringstream<_CharT, _Traits> __ostr;
__ostr << __str._M_delim;
for (const _CharT* __c = __str._M_string; *__c; ++__c)
{
if (*__c == __str._M_delim || *__c == __str._M_escape)
__ostr << __str._M_escape;
__ostr << *__c;
}
__ostr << __str._M_delim;
return __os << __ostr.str();
}
template<typename _CharT, typename _Traits, typename _String>
std::basic_ostream<_CharT, _Traits>&
operator<<(std::basic_ostream<_CharT, _Traits>& __os,
const _Quoted_string<_String, _CharT>& __str)
{
std::basic_ostringstream<_CharT, _Traits> __ostr;
__ostr << __str._M_delim;
for (auto __c : __str._M_string)
{
if (__c == __str._M_delim || __c == __str._M_escape)
__ostr << __str._M_escape;
__ostr << __c;
}
__ostr << __str._M_delim;
return __os << __ostr.str();
}
template<typename _CharT, typename _Traits, typename _Alloc>
std::basic_istream<_CharT, _Traits>&
operator>>(std::basic_istream<_CharT, _Traits>& __is,
const _Quoted_string<basic_string<_CharT, _Traits, _Alloc>&,
_CharT>& __str)
{
_CharT __c;
__is >> __c;
if (!__is.good())
return __is;
if (__c != __str._M_delim)
{
__is.unget();
__is >> __str._M_string;
return __is;
}
__str._M_string.clear();
std::ios_base::fmtflags __flags
= __is.flags(__is.flags() & ~std::ios_base::skipws);
do
{
__is >> __c;
if (!__is.good())
break;
if (__c == __str._M_escape)
{
__is >> __c;
if (!__is.good())
break;
}
else if (__c == __str._M_delim)
break;
__str._M_string += __c;
}
while (true);
__is.setf(__flags);
return __is;
}
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
struct _Resetiosflags { ios_base::fmtflags _M_mask; };
inline _Resetiosflags
resetiosflags(ios_base::fmtflags __mask)
{ return { __mask }; }
template<typename _CharT, typename _Traits>
inline basic_istream<_CharT, _Traits>&
operator>>(basic_istream<_CharT, _Traits>& __is, _Resetiosflags __f)
{
__is.setf(ios_base::fmtflags(0), __f._M_mask);
return __is;
}
template<typename _CharT, typename _Traits>
inline basic_ostream<_CharT, _Traits>&
operator<<(basic_ostream<_CharT, _Traits>& __os, _Resetiosflags __f)
{
__os.setf(ios_base::fmtflags(0), __f._M_mask);
return __os;
}
struct _Setiosflags { ios_base::fmtflags _M_mask; };
inline _Setiosflags
setiosflags(ios_base::fmtflags __mask)
{ return { __mask }; }
template<typename _CharT, typename _Traits>
inline basic_istream<_CharT, _Traits>&
operator>>(basic_istream<_CharT, _Traits>& __is, _Setiosflags __f)
{
__is.setf(__f._M_mask);
return __is;
}
template<typename _CharT, typename _Traits>
inline basic_ostream<_CharT, _Traits>&
operator<<(basic_ostream<_CharT, _Traits>& __os, _Setiosflags __f)
{
__os.setf(__f._M_mask);
return __os;
}
struct _Setbase { int _M_base; };
inline _Setbase
setbase(int __base)
{ return { __base }; }
template<typename _CharT, typename _Traits>
inline basic_istream<_CharT, _Traits>&
operator>>(basic_istream<_CharT, _Traits>& __is, _Setbase __f)
{
__is.setf(__f._M_base == 8 ? ios_base::oct :
__f._M_base == 10 ? ios_base::dec :
__f._M_base == 16 ? ios_base::hex :
ios_base::fmtflags(0), ios_base::basefield);
return __is;
}
template<typename _CharT, typename _Traits>
inline basic_ostream<_CharT, _Traits>&
operator<<(basic_ostream<_CharT, _Traits>& __os, _Setbase __f)
{
__os.setf(__f._M_base == 8 ? ios_base::oct :
__f._M_base == 10 ? ios_base::dec :
__f._M_base == 16 ? ios_base::hex :
ios_base::fmtflags(0), ios_base::basefield);
return __os;
}
template<typename _CharT>
struct _Setfill { _CharT _M_c; };
template<typename _CharT>
inline _Setfill<_CharT>
setfill(_CharT __c)
{ return { __c }; }
template<typename _CharT, typename _Traits>
inline basic_istream<_CharT, _Traits>&
operator>>(basic_istream<_CharT, _Traits>& __is, _Setfill<_CharT> __f)
{
__is.fill(__f._M_c);
return __is;
}
template<typename _CharT, typename _Traits>
inline basic_ostream<_CharT, _Traits>&
operator<<(basic_ostream<_CharT, _Traits>& __os, _Setfill<_CharT> __f)
{
__os.fill(__f._M_c);
return __os;
}
struct _Setprecision { int _M_n; };
inline _Setprecision
setprecision(int __n)
{ return { __n }; }
template<typename _CharT, typename _Traits>
inline basic_istream<_CharT, _Traits>&
operator>>(basic_istream<_CharT, _Traits>& __is, _Setprecision __f)
{
__is.precision(__f._M_n);
return __is;
}
template<typename _CharT, typename _Traits>
inline basic_ostream<_CharT, _Traits>&
operator<<(basic_ostream<_CharT, _Traits>& __os, _Setprecision __f)
{
__os.precision(__f._M_n);
return __os;
}
struct _Setw { int _M_n; };
inline _Setw
setw(int __n)
{ return { __n }; }
template<typename _CharT, typename _Traits>
inline basic_istream<_CharT, _Traits>&
operator>>(basic_istream<_CharT, _Traits>& __is, _Setw __f)
{
__is.width(__f._M_n);
return __is;
}
template<typename _CharT, typename _Traits>
inline basic_ostream<_CharT, _Traits>&
operator<<(basic_ostream<_CharT, _Traits>& __os, _Setw __f)
{
__os.width(__f._M_n);
return __os;
}
template<typename _MoneyT>
struct _Get_money { _MoneyT& _M_mon; bool _M_intl; };
template<typename _MoneyT>
inline _Get_money<_MoneyT>
get_money(_MoneyT& __mon, bool __intl = false)
{ return { __mon, __intl }; }
template<typename _CharT, typename _Traits, typename _MoneyT>
basic_istream<_CharT, _Traits>&
operator>>(basic_istream<_CharT, _Traits>& __is, _Get_money<_MoneyT> __f)
{
typename basic_istream<_CharT, _Traits>::sentry __cerb(__is, false);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
typedef istreambuf_iterator<_CharT, _Traits> _Iter;
typedef money_get<_CharT, _Iter> _MoneyGet;
const _MoneyGet& __mg = use_facet<_MoneyGet>(__is.getloc());
__mg.get(_Iter(__is.rdbuf()), _Iter(), __f._M_intl,
__is, __err, __f._M_mon);
}
catch(__cxxabiv1::__forced_unwind&)
{
__is._M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ __is._M_setstate(ios_base::badbit); }
if (__err)
__is.setstate(__err);
}
return __is;
}
template<typename _MoneyT>
struct _Put_money { const _MoneyT& _M_mon; bool _M_intl; };
template<typename _MoneyT>
inline _Put_money<_MoneyT>
put_money(const _MoneyT& __mon, bool __intl = false)
{ return { __mon, __intl }; }
template<typename _CharT, typename _Traits, typename _MoneyT>
basic_ostream<_CharT, _Traits>&
operator<<(basic_ostream<_CharT, _Traits>& __os, _Put_money<_MoneyT> __f)
{
typename basic_ostream<_CharT, _Traits>::sentry __cerb(__os);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
typedef ostreambuf_iterator<_CharT, _Traits> _Iter;
typedef money_put<_CharT, _Iter> _MoneyPut;
const _MoneyPut& __mp = use_facet<_MoneyPut>(__os.getloc());
if (__mp.put(_Iter(__os.rdbuf()), __f._M_intl, __os,
__os.fill(), __f._M_mon).failed())
__err |= ios_base::badbit;
}
catch(__cxxabiv1::__forced_unwind&)
{
__os._M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ __os._M_setstate(ios_base::badbit); }
if (__err)
__os.setstate(__err);
}
return __os;
}
template<typename _CharT>
struct _Put_time
{
const std::tm* _M_tmb;
const _CharT* _M_fmt;
};
template<typename _CharT>
inline _Put_time<_CharT>
put_time(const std::tm* __tmb, const _CharT* __fmt)
{ return { __tmb, __fmt }; }
template<typename _CharT, typename _Traits>
basic_ostream<_CharT, _Traits>&
operator<<(basic_ostream<_CharT, _Traits>& __os, _Put_time<_CharT> __f)
{
typename basic_ostream<_CharT, _Traits>::sentry __cerb(__os);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
typedef ostreambuf_iterator<_CharT, _Traits> _Iter;
typedef time_put<_CharT, _Iter> _TimePut;
const _CharT* const __fmt_end = __f._M_fmt +
_Traits::length(__f._M_fmt);
const _TimePut& __mp = use_facet<_TimePut>(__os.getloc());
if (__mp.put(_Iter(__os.rdbuf()), __os, __os.fill(),
__f._M_tmb, __f._M_fmt, __fmt_end).failed())
__err |= ios_base::badbit;
}
catch(__cxxabiv1::__forced_unwind&)
{
__os._M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ __os._M_setstate(ios_base::badbit); }
if (__err)
__os.setstate(__err);
}
return __os;
}
template<typename _CharT>
struct _Get_time
{
std::tm* _M_tmb;
const _CharT* _M_fmt;
};
template<typename _CharT>
inline _Get_time<_CharT>
get_time(std::tm* __tmb, const _CharT* __fmt)
{ return { __tmb, __fmt }; }
template<typename _CharT, typename _Traits>
basic_istream<_CharT, _Traits>&
operator>>(basic_istream<_CharT, _Traits>& __is, _Get_time<_CharT> __f)
{
typename basic_istream<_CharT, _Traits>::sentry __cerb(__is, false);
if (__cerb)
{
ios_base::iostate __err = ios_base::goodbit;
try
{
typedef istreambuf_iterator<_CharT, _Traits> _Iter;
typedef time_get<_CharT, _Iter> _TimeGet;
const _CharT* const __fmt_end = __f._M_fmt +
_Traits::length(__f._M_fmt);
const _TimeGet& __mg = use_facet<_TimeGet>(__is.getloc());
__mg.get(_Iter(__is.rdbuf()), _Iter(), __is,
__err, __f._M_tmb, __f._M_fmt, __fmt_end);
}
catch(__cxxabiv1::__forced_unwind&)
{
__is._M_setstate(ios_base::badbit);
throw;
}
catch(...)
{ __is._M_setstate(ios_base::badbit); }
if (__err)
__is.setstate(__err);
}
return __is;
}
template<typename _CharT>
inline auto
quoted(const _CharT* __string,
_CharT __delim = _CharT('"'), _CharT __escape = _CharT('\\'))
{
return __detail::_Quoted_string<const _CharT*, _CharT>(__string, __delim,
__escape);
}
template<typename _CharT, typename _Traits, typename _Alloc>
inline auto
quoted(const basic_string<_CharT, _Traits, _Alloc>& __string,
_CharT __delim = _CharT('"'), _CharT __escape = _CharT('\\'))
{
return __detail::_Quoted_string<
const basic_string<_CharT, _Traits, _Alloc>&, _CharT>(
__string, __delim, __escape);
}
template<typename _CharT, typename _Traits, typename _Alloc>
inline auto
quoted(basic_string<_CharT, _Traits, _Alloc>& __string,
_CharT __delim = _CharT('"'), _CharT __escape = _CharT('\\'))
{
return __detail::_Quoted_string<
basic_string<_CharT, _Traits, _Alloc>&, _CharT>(
__string, __delim, __escape);
}
template<typename _CharT, typename _Traits>
inline auto
quoted(basic_string_view<_CharT, _Traits> __sv,
_CharT __delim = _CharT('"'), _CharT __escape = _CharT('\\'))
{
return __detail::_Quoted_string<
basic_string_view<_CharT, _Traits>, _CharT>(__sv, __delim, __escape);
}
extern template ostream& operator<<(ostream&, _Setfill<char>);
extern template ostream& operator<<(ostream&, _Setiosflags);
extern template ostream& operator<<(ostream&, _Resetiosflags);
extern template ostream& operator<<(ostream&, _Setbase);
extern template ostream& operator<<(ostream&, _Setprecision);
extern template ostream& operator<<(ostream&, _Setw);
extern template istream& operator>>(istream&, _Setfill<char>);
extern template istream& operator>>(istream&, _Setiosflags);
extern template istream& operator>>(istream&, _Resetiosflags);
extern template istream& operator>>(istream&, _Setbase);
extern template istream& operator>>(istream&, _Setprecision);
extern template istream& operator>>(istream&, _Setw);
extern template wostream& operator<<(wostream&, _Setfill<wchar_t>);
extern template wostream& operator<<(wostream&, _Setiosflags);
extern template wostream& operator<<(wostream&, _Resetiosflags);
extern template wostream& operator<<(wostream&, _Setbase);
extern template wostream& operator<<(wostream&, _Setprecision);
extern template wostream& operator<<(wostream&, _Setw);
extern template wistream& operator>>(wistream&, _Setfill<wchar_t>);
extern template wistream& operator>>(wistream&, _Setiosflags);
extern template wistream& operator>>(wistream&, _Resetiosflags);
extern template wistream& operator>>(wistream&, _Setbase);
extern template wistream& operator>>(wistream&, _Setprecision);
extern template wistream& operator>>(wistream&, _Setw);
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Val, typename _NodeAlloc>
class _Node_handle_common
{
using _AllocTraits = allocator_traits<_NodeAlloc>;
public:
using allocator_type = __alloc_rebind<_NodeAlloc, _Val>;
allocator_type
get_allocator() const noexcept
{
do { if (std::__is_constant_evaluated() && !bool(!this->empty())) __builtin_unreachable(); } while (false);
return allocator_type(_M_alloc._M_alloc);
}
explicit operator bool() const noexcept { return _M_ptr != nullptr; }
[[nodiscard]] bool empty() const noexcept { return _M_ptr == nullptr; }
protected:
constexpr _Node_handle_common() noexcept : _M_ptr() { }
~_Node_handle_common()
{
if (!empty())
_M_reset();
}
_Node_handle_common(_Node_handle_common&& __nh) noexcept
: _M_ptr(__nh._M_ptr)
{
if (_M_ptr)
_M_move(std::move(__nh));
}
_Node_handle_common&
operator=(_Node_handle_common&& __nh) noexcept
{
if (empty())
{
if (!__nh.empty())
_M_move(std::move(__nh));
}
else if (__nh.empty())
_M_reset();
else
{
_AllocTraits::destroy(*_M_alloc, _M_ptr->_M_valptr());
_AllocTraits::deallocate(*_M_alloc, _M_ptr, 1);
_M_alloc = __nh._M_alloc.release();
_M_ptr = __nh._M_ptr;
__nh._M_ptr = nullptr;
}
return *this;
}
_Node_handle_common(typename _AllocTraits::pointer __ptr,
const _NodeAlloc& __alloc)
: _M_ptr(__ptr), _M_alloc(__alloc)
{
do { if (std::__is_constant_evaluated() && !bool(__ptr != nullptr)) __builtin_unreachable(); } while (false);
}
void
_M_swap(_Node_handle_common& __nh) noexcept
{
if (empty())
{
if (!__nh.empty())
_M_move(std::move(__nh));
}
else if (__nh.empty())
__nh._M_move(std::move(*this));
else
{
using std::swap;
swap(_M_ptr, __nh._M_ptr);
_M_alloc.swap(__nh._M_alloc);
}
}
private:
void
_M_move(_Node_handle_common&& __nh) noexcept
{
::new (std::__addressof(_M_alloc)) _NodeAlloc(__nh._M_alloc.release());
_M_ptr = __nh._M_ptr;
__nh._M_ptr = nullptr;
}
void
_M_reset() noexcept
{
_NodeAlloc __alloc = _M_alloc.release();
_AllocTraits::destroy(__alloc, _M_ptr->_M_valptr());
_AllocTraits::deallocate(__alloc, _M_ptr, 1);
_M_ptr = nullptr;
}
protected:
typename _AllocTraits::pointer _M_ptr;
private:
union _Optional_alloc
{
_Optional_alloc() { }
~_Optional_alloc() { }
_Optional_alloc(_Optional_alloc&&) = delete;
_Optional_alloc& operator=(_Optional_alloc&&) = delete;
_Optional_alloc(const _NodeAlloc& __alloc) noexcept
: _M_alloc(__alloc)
{ }
void
operator=(_NodeAlloc&& __alloc) noexcept
{
using _ATr = _AllocTraits;
if constexpr (_ATr::propagate_on_container_move_assignment::value)
_M_alloc = std::move(__alloc);
else if constexpr (!_AllocTraits::is_always_equal::value)
do { if (std::__is_constant_evaluated() && !bool(_M_alloc == __alloc)) __builtin_unreachable(); } while (false);
}
void
swap(_Optional_alloc& __other) noexcept
{
using std::swap;
if constexpr (_AllocTraits::propagate_on_container_swap::value)
swap(_M_alloc, __other._M_alloc);
else if constexpr (!_AllocTraits::is_always_equal::value)
do { if (std::__is_constant_evaluated() && !bool(_M_alloc == __other._M_alloc)) __builtin_unreachable(); } while (false);
}
_NodeAlloc& operator*() noexcept { return _M_alloc; }
_NodeAlloc release() noexcept
{
_NodeAlloc __tmp = std::move(_M_alloc);
_M_alloc.~_NodeAlloc();
return __tmp;
}
struct _Empty { };
[[__no_unique_address__]] _Empty _M_empty;
[[__no_unique_address__]] _NodeAlloc _M_alloc;
};
[[__no_unique_address__]] _Optional_alloc _M_alloc;
template<typename _Key2, typename _Value2, typename _KeyOfValue,
typename _Compare, typename _ValueAlloc>
friend class _Rb_tree;
};
template<typename _Key, typename _Value, typename _NodeAlloc>
class _Node_handle : public _Node_handle_common<_Value, _NodeAlloc>
{
public:
constexpr _Node_handle() noexcept = default;
~_Node_handle() = default;
_Node_handle(_Node_handle&&) noexcept = default;
_Node_handle&
operator=(_Node_handle&&) noexcept = default;
using key_type = _Key;
using mapped_type = typename _Value::second_type;
key_type&
key() const noexcept
{
do { if (std::__is_constant_evaluated() && !bool(!this->empty())) __builtin_unreachable(); } while (false);
return *_M_pkey;
}
mapped_type&
mapped() const noexcept
{
do { if (std::__is_constant_evaluated() && !bool(!this->empty())) __builtin_unreachable(); } while (false);
return *_M_pmapped;
}
void
swap(_Node_handle& __nh) noexcept
{
this->_M_swap(__nh);
using std::swap;
swap(_M_pkey, __nh._M_pkey);
swap(_M_pmapped, __nh._M_pmapped);
}
friend void
swap(_Node_handle& __x, _Node_handle& __y)
noexcept(noexcept(__x.swap(__y)))
{ __x.swap(__y); }
private:
using _AllocTraits = allocator_traits<_NodeAlloc>;
_Node_handle(typename _AllocTraits::pointer __ptr,
const _NodeAlloc& __alloc)
: _Node_handle_common<_Value, _NodeAlloc>(__ptr, __alloc)
{
if (__ptr)
{
auto& __key = const_cast<_Key&>(__ptr->_M_valptr()->first);
_M_pkey = _S_pointer_to(__key);
_M_pmapped = _S_pointer_to(__ptr->_M_valptr()->second);
}
else
{
_M_pkey = nullptr;
_M_pmapped = nullptr;
}
}
template<typename _Tp>
using __pointer
= __ptr_rebind<typename _AllocTraits::pointer,
remove_reference_t<_Tp>>;
__pointer<_Key> _M_pkey = nullptr;
__pointer<typename _Value::second_type> _M_pmapped = nullptr;
template<typename _Tp>
__pointer<_Tp>
_S_pointer_to(_Tp& __obj)
{ return pointer_traits<__pointer<_Tp>>::pointer_to(__obj); }
const key_type&
_M_key() const noexcept { return key(); }
template<typename _Key2, typename _Value2, typename _KeyOfValue,
typename _Compare, typename _ValueAlloc>
friend class _Rb_tree;
template<typename _Key2, typename _Value2, typename _ValueAlloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
friend class _Hashtable;
};
template<typename _Value, typename _NodeAlloc>
class _Node_handle<_Value, _Value, _NodeAlloc>
: public _Node_handle_common<_Value, _NodeAlloc>
{
public:
constexpr _Node_handle() noexcept = default;
~_Node_handle() = default;
_Node_handle(_Node_handle&&) noexcept = default;
_Node_handle&
operator=(_Node_handle&&) noexcept = default;
using value_type = _Value;
value_type&
value() const noexcept
{
do { if (std::__is_constant_evaluated() && !bool(!this->empty())) __builtin_unreachable(); } while (false);
return *this->_M_ptr->_M_valptr();
}
void
swap(_Node_handle& __nh) noexcept
{ this->_M_swap(__nh); }
friend void
swap(_Node_handle& __x, _Node_handle& __y)
noexcept(noexcept(__x.swap(__y)))
{ __x.swap(__y); }
private:
using _AllocTraits = allocator_traits<_NodeAlloc>;
_Node_handle(typename _AllocTraits::pointer __ptr,
const _NodeAlloc& __alloc)
: _Node_handle_common<_Value, _NodeAlloc>(__ptr, __alloc) { }
const value_type&
_M_key() const noexcept { return value(); }
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
friend class _Rb_tree;
template<typename _Key2, typename _Value2, typename _ValueAlloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
friend class _Hashtable;
};
template<typename _Iterator, typename _NodeHandle>
struct _Node_insert_return
{
_Iterator position = _Iterator();
bool inserted = false;
_NodeHandle node;
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
enum _Rb_tree_color { _S_red = false, _S_black = true };
struct _Rb_tree_node_base
{
typedef _Rb_tree_node_base* _Base_ptr;
typedef const _Rb_tree_node_base* _Const_Base_ptr;
_Rb_tree_color _M_color;
_Base_ptr _M_parent;
_Base_ptr _M_left;
_Base_ptr _M_right;
static _Base_ptr
_S_minimum(_Base_ptr __x) noexcept
{
while (__x->_M_left != 0) __x = __x->_M_left;
return __x;
}
static _Const_Base_ptr
_S_minimum(_Const_Base_ptr __x) noexcept
{
while (__x->_M_left != 0) __x = __x->_M_left;
return __x;
}
static _Base_ptr
_S_maximum(_Base_ptr __x) noexcept
{
while (__x->_M_right != 0) __x = __x->_M_right;
return __x;
}
static _Const_Base_ptr
_S_maximum(_Const_Base_ptr __x) noexcept
{
while (__x->_M_right != 0) __x = __x->_M_right;
return __x;
}
};
template<typename _Key_compare>
struct _Rb_tree_key_compare
{
_Key_compare _M_key_compare;
_Rb_tree_key_compare()
noexcept(is_nothrow_default_constructible<_Key_compare>::value)
: _M_key_compare()
{ }
_Rb_tree_key_compare(const _Key_compare& __comp)
: _M_key_compare(__comp)
{ }
_Rb_tree_key_compare(const _Rb_tree_key_compare&) = default;
_Rb_tree_key_compare(_Rb_tree_key_compare&& __x)
noexcept(is_nothrow_copy_constructible<_Key_compare>::value)
: _M_key_compare(__x._M_key_compare)
{ }
};
struct _Rb_tree_header
{
_Rb_tree_node_base _M_header;
size_t _M_node_count;
_Rb_tree_header() noexcept
{
_M_header._M_color = _S_red;
_M_reset();
}
_Rb_tree_header(_Rb_tree_header&& __x) noexcept
{
if (__x._M_header._M_parent != nullptr)
_M_move_data(__x);
else
{
_M_header._M_color = _S_red;
_M_reset();
}
}
void
_M_move_data(_Rb_tree_header& __from)
{
_M_header._M_color = __from._M_header._M_color;
_M_header._M_parent = __from._M_header._M_parent;
_M_header._M_left = __from._M_header._M_left;
_M_header._M_right = __from._M_header._M_right;
_M_header._M_parent->_M_parent = &_M_header;
_M_node_count = __from._M_node_count;
__from._M_reset();
}
void
_M_reset()
{
_M_header._M_parent = 0;
_M_header._M_left = &_M_header;
_M_header._M_right = &_M_header;
_M_node_count = 0;
}
};
template<typename _Val>
struct _Rb_tree_node : public _Rb_tree_node_base
{
typedef _Rb_tree_node<_Val>* _Link_type;
__gnu_cxx::__aligned_membuf<_Val> _M_storage;
_Val*
_M_valptr()
{ return _M_storage._M_ptr(); }
const _Val*
_M_valptr() const
{ return _M_storage._M_ptr(); }
};
__attribute__ ((__pure__)) _Rb_tree_node_base*
_Rb_tree_increment(_Rb_tree_node_base* __x) throw ();
__attribute__ ((__pure__)) const _Rb_tree_node_base*
_Rb_tree_increment(const _Rb_tree_node_base* __x) throw ();
__attribute__ ((__pure__)) _Rb_tree_node_base*
_Rb_tree_decrement(_Rb_tree_node_base* __x) throw ();
__attribute__ ((__pure__)) const _Rb_tree_node_base*
_Rb_tree_decrement(const _Rb_tree_node_base* __x) throw ();
template<typename _Tp>
struct _Rb_tree_iterator
{
typedef _Tp value_type;
typedef _Tp& reference;
typedef _Tp* pointer;
typedef bidirectional_iterator_tag iterator_category;
typedef ptrdiff_t difference_type;
typedef _Rb_tree_iterator<_Tp> _Self;
typedef _Rb_tree_node_base::_Base_ptr _Base_ptr;
typedef _Rb_tree_node<_Tp>* _Link_type;
_Rb_tree_iterator() noexcept
: _M_node() { }
explicit
_Rb_tree_iterator(_Base_ptr __x) noexcept
: _M_node(__x) { }
reference
operator*() const noexcept
{ return *static_cast<_Link_type>(_M_node)->_M_valptr(); }
pointer
operator->() const noexcept
{ return static_cast<_Link_type> (_M_node)->_M_valptr(); }
_Self&
operator++() noexcept
{
_M_node = _Rb_tree_increment(_M_node);
return *this;
}
_Self
operator++(int) noexcept
{
_Self __tmp = *this;
_M_node = _Rb_tree_increment(_M_node);
return __tmp;
}
_Self&
operator--() noexcept
{
_M_node = _Rb_tree_decrement(_M_node);
return *this;
}
_Self
operator--(int) noexcept
{
_Self __tmp = *this;
_M_node = _Rb_tree_decrement(_M_node);
return __tmp;
}
friend bool
operator==(const _Self& __x, const _Self& __y) noexcept
{ return __x._M_node == __y._M_node; }
friend bool
operator!=(const _Self& __x, const _Self& __y) noexcept
{ return __x._M_node != __y._M_node; }
_Base_ptr _M_node;
};
template<typename _Tp>
struct _Rb_tree_const_iterator
{
typedef _Tp value_type;
typedef const _Tp& reference;
typedef const _Tp* pointer;
typedef _Rb_tree_iterator<_Tp> iterator;
typedef bidirectional_iterator_tag iterator_category;
typedef ptrdiff_t difference_type;
typedef _Rb_tree_const_iterator<_Tp> _Self;
typedef _Rb_tree_node_base::_Const_Base_ptr _Base_ptr;
typedef const _Rb_tree_node<_Tp>* _Link_type;
_Rb_tree_const_iterator() noexcept
: _M_node() { }
explicit
_Rb_tree_const_iterator(_Base_ptr __x) noexcept
: _M_node(__x) { }
_Rb_tree_const_iterator(const iterator& __it) noexcept
: _M_node(__it._M_node) { }
iterator
_M_const_cast() const noexcept
{ return iterator(const_cast<typename iterator::_Base_ptr>(_M_node)); }
reference
operator*() const noexcept
{ return *static_cast<_Link_type>(_M_node)->_M_valptr(); }
pointer
operator->() const noexcept
{ return static_cast<_Link_type>(_M_node)->_M_valptr(); }
_Self&
operator++() noexcept
{
_M_node = _Rb_tree_increment(_M_node);
return *this;
}
_Self
operator++(int) noexcept
{
_Self __tmp = *this;
_M_node = _Rb_tree_increment(_M_node);
return __tmp;
}
_Self&
operator--() noexcept
{
_M_node = _Rb_tree_decrement(_M_node);
return *this;
}
_Self
operator--(int) noexcept
{
_Self __tmp = *this;
_M_node = _Rb_tree_decrement(_M_node);
return __tmp;
}
friend bool
operator==(const _Self& __x, const _Self& __y) noexcept
{ return __x._M_node == __y._M_node; }
friend bool
operator!=(const _Self& __x, const _Self& __y) noexcept
{ return __x._M_node != __y._M_node; }
_Base_ptr _M_node;
};
__attribute__((__nonnull__))
void
_Rb_tree_insert_and_rebalance(const bool __insert_left,
_Rb_tree_node_base* __x,
_Rb_tree_node_base* __p,
_Rb_tree_node_base& __header) throw ();
__attribute__((__nonnull__,__returns_nonnull__))
_Rb_tree_node_base*
_Rb_tree_rebalance_for_erase(_Rb_tree_node_base* const __z,
_Rb_tree_node_base& __header) throw ();
template<typename _Tree1, typename _Cmp2>
struct _Rb_tree_merge_helper { };
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc = allocator<_Val> >
class _Rb_tree
{
typedef typename __gnu_cxx::__alloc_traits<_Alloc>::template
rebind<_Rb_tree_node<_Val> >::other _Node_allocator;
typedef __gnu_cxx::__alloc_traits<_Node_allocator> _Alloc_traits;
protected:
typedef _Rb_tree_node_base* _Base_ptr;
typedef const _Rb_tree_node_base* _Const_Base_ptr;
typedef _Rb_tree_node<_Val>* _Link_type;
typedef const _Rb_tree_node<_Val>* _Const_Link_type;
private:
struct _Reuse_or_alloc_node
{
_Reuse_or_alloc_node(_Rb_tree& __t)
: _M_root(__t._M_root()), _M_nodes(__t._M_rightmost()), _M_t(__t)
{
if (_M_root)
{
_M_root->_M_parent = 0;
if (_M_nodes->_M_left)
_M_nodes = _M_nodes->_M_left;
}
else
_M_nodes = 0;
}
_Reuse_or_alloc_node(const _Reuse_or_alloc_node&) = delete;
~_Reuse_or_alloc_node()
{ _M_t._M_erase(static_cast<_Link_type>(_M_root)); }
template<typename _Arg>
_Link_type
operator()(_Arg&& __arg)
{
_Link_type __node = static_cast<_Link_type>(_M_extract());
if (__node)
{
_M_t._M_destroy_node(__node);
_M_t._M_construct_node(__node, std::forward<_Arg>(__arg));
return __node;
}
return _M_t._M_create_node(std::forward<_Arg>(__arg));
}
private:
_Base_ptr
_M_extract()
{
if (!_M_nodes)
return _M_nodes;
_Base_ptr __node = _M_nodes;
_M_nodes = _M_nodes->_M_parent;
if (_M_nodes)
{
if (_M_nodes->_M_right == __node)
{
_M_nodes->_M_right = 0;
if (_M_nodes->_M_left)
{
_M_nodes = _M_nodes->_M_left;
while (_M_nodes->_M_right)
_M_nodes = _M_nodes->_M_right;
if (_M_nodes->_M_left)
_M_nodes = _M_nodes->_M_left;
}
}
else
_M_nodes->_M_left = 0;
}
else
_M_root = 0;
return __node;
}
_Base_ptr _M_root;
_Base_ptr _M_nodes;
_Rb_tree& _M_t;
};
struct _Alloc_node
{
_Alloc_node(_Rb_tree& __t)
: _M_t(__t) { }
template<typename _Arg>
_Link_type
operator()(_Arg&& __arg) const
{ return _M_t._M_create_node(std::forward<_Arg>(__arg)); }
private:
_Rb_tree& _M_t;
};
public:
typedef _Key key_type;
typedef _Val value_type;
typedef value_type* pointer;
typedef const value_type* const_pointer;
typedef value_type& reference;
typedef const value_type& const_reference;
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef _Alloc allocator_type;
_Node_allocator&
_M_get_Node_allocator() noexcept
{ return this->_M_impl; }
const _Node_allocator&
_M_get_Node_allocator() const noexcept
{ return this->_M_impl; }
allocator_type
get_allocator() const noexcept
{ return allocator_type(_M_get_Node_allocator()); }
protected:
_Link_type
_M_get_node()
{ return _Alloc_traits::allocate(_M_get_Node_allocator(), 1); }
void
_M_put_node(_Link_type __p) noexcept
{ _Alloc_traits::deallocate(_M_get_Node_allocator(), __p, 1); }
template<typename... _Args>
void
_M_construct_node(_Link_type __node, _Args&&... __args)
{
try
{
::new(__node) _Rb_tree_node<_Val>;
_Alloc_traits::construct(_M_get_Node_allocator(),
__node->_M_valptr(),
std::forward<_Args>(__args)...);
}
catch(...)
{
__node->~_Rb_tree_node<_Val>();
_M_put_node(__node);
throw;
}
}
template<typename... _Args>
_Link_type
_M_create_node(_Args&&... __args)
{
_Link_type __tmp = _M_get_node();
_M_construct_node(__tmp, std::forward<_Args>(__args)...);
return __tmp;
}
void
_M_destroy_node(_Link_type __p) noexcept
{
_Alloc_traits::destroy(_M_get_Node_allocator(), __p->_M_valptr());
__p->~_Rb_tree_node<_Val>();
}
void
_M_drop_node(_Link_type __p) noexcept
{
_M_destroy_node(__p);
_M_put_node(__p);
}
template<bool _MoveValue, typename _NodeGen>
_Link_type
_M_clone_node(_Link_type __x, _NodeGen& __node_gen)
{
using _Vp = __conditional_t<_MoveValue,
value_type&&,
const value_type&>;
_Link_type __tmp
= __node_gen(std::forward<_Vp>(*__x->_M_valptr()));
__tmp->_M_color = __x->_M_color;
__tmp->_M_left = 0;
__tmp->_M_right = 0;
return __tmp;
}
protected:
template<typename _Key_compare,
bool = __is_pod(_Key_compare)>
struct _Rb_tree_impl
: public _Node_allocator
, public _Rb_tree_key_compare<_Key_compare>
, public _Rb_tree_header
{
typedef _Rb_tree_key_compare<_Key_compare> _Base_key_compare;
_Rb_tree_impl()
noexcept(is_nothrow_default_constructible<_Node_allocator>::value && is_nothrow_default_constructible<_Base_key_compare>::value)
: _Node_allocator()
{ }
_Rb_tree_impl(const _Rb_tree_impl& __x)
: _Node_allocator(_Alloc_traits::_S_select_on_copy(__x))
, _Base_key_compare(__x._M_key_compare)
, _Rb_tree_header()
{ }
_Rb_tree_impl(_Rb_tree_impl&&)
noexcept( is_nothrow_move_constructible<_Base_key_compare>::value )
= default;
explicit
_Rb_tree_impl(_Node_allocator&& __a)
: _Node_allocator(std::move(__a))
{ }
_Rb_tree_impl(_Rb_tree_impl&& __x, _Node_allocator&& __a)
: _Node_allocator(std::move(__a)),
_Base_key_compare(std::move(__x)),
_Rb_tree_header(std::move(__x))
{ }
_Rb_tree_impl(const _Key_compare& __comp, _Node_allocator&& __a)
: _Node_allocator(std::move(__a)), _Base_key_compare(__comp)
{ }
};
_Rb_tree_impl<_Compare> _M_impl;
protected:
_Base_ptr&
_M_root() noexcept
{ return this->_M_impl._M_header._M_parent; }
_Const_Base_ptr
_M_root() const noexcept
{ return this->_M_impl._M_header._M_parent; }
_Base_ptr&
_M_leftmost() noexcept
{ return this->_M_impl._M_header._M_left; }
_Const_Base_ptr
_M_leftmost() const noexcept
{ return this->_M_impl._M_header._M_left; }
_Base_ptr&
_M_rightmost() noexcept
{ return this->_M_impl._M_header._M_right; }
_Const_Base_ptr
_M_rightmost() const noexcept
{ return this->_M_impl._M_header._M_right; }
_Link_type
_M_mbegin() const noexcept
{ return static_cast<_Link_type>(this->_M_impl._M_header._M_parent); }
_Link_type
_M_begin() noexcept
{ return _M_mbegin(); }
_Const_Link_type
_M_begin() const noexcept
{
return static_cast<_Const_Link_type>
(this->_M_impl._M_header._M_parent);
}
_Base_ptr
_M_end() noexcept
{ return &this->_M_impl._M_header; }
_Const_Base_ptr
_M_end() const noexcept
{ return &this->_M_impl._M_header; }
static const _Key&
_S_key(_Const_Link_type __x)
{
static_assert(__is_invocable<_Compare&, const _Key&, const _Key&>{},
"comparison object must be invocable "
"with two arguments of key type");
if constexpr (__is_invocable<_Compare&, const _Key&, const _Key&>{})
static_assert(
is_invocable_v<const _Compare&, const _Key&, const _Key&>,
"comparison object must be invocable as const");
return _KeyOfValue()(*__x->_M_valptr());
}
static _Link_type
_S_left(_Base_ptr __x) noexcept
{ return static_cast<_Link_type>(__x->_M_left); }
static _Const_Link_type
_S_left(_Const_Base_ptr __x) noexcept
{ return static_cast<_Const_Link_type>(__x->_M_left); }
static _Link_type
_S_right(_Base_ptr __x) noexcept
{ return static_cast<_Link_type>(__x->_M_right); }
static _Const_Link_type
_S_right(_Const_Base_ptr __x) noexcept
{ return static_cast<_Const_Link_type>(__x->_M_right); }
static const _Key&
_S_key(_Const_Base_ptr __x)
{ return _S_key(static_cast<_Const_Link_type>(__x)); }
static _Base_ptr
_S_minimum(_Base_ptr __x) noexcept
{ return _Rb_tree_node_base::_S_minimum(__x); }
static _Const_Base_ptr
_S_minimum(_Const_Base_ptr __x) noexcept
{ return _Rb_tree_node_base::_S_minimum(__x); }
static _Base_ptr
_S_maximum(_Base_ptr __x) noexcept
{ return _Rb_tree_node_base::_S_maximum(__x); }
static _Const_Base_ptr
_S_maximum(_Const_Base_ptr __x) noexcept
{ return _Rb_tree_node_base::_S_maximum(__x); }
public:
typedef _Rb_tree_iterator<value_type> iterator;
typedef _Rb_tree_const_iterator<value_type> const_iterator;
typedef std::reverse_iterator<iterator> reverse_iterator;
typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
using node_type = _Node_handle<_Key, _Val, _Node_allocator>;
using insert_return_type = _Node_insert_return<
__conditional_t<is_same_v<_Key, _Val>, const_iterator, iterator>,
node_type>;
pair<_Base_ptr, _Base_ptr>
_M_get_insert_unique_pos(const key_type& __k);
pair<_Base_ptr, _Base_ptr>
_M_get_insert_equal_pos(const key_type& __k);
pair<_Base_ptr, _Base_ptr>
_M_get_insert_hint_unique_pos(const_iterator __pos,
const key_type& __k);
pair<_Base_ptr, _Base_ptr>
_M_get_insert_hint_equal_pos(const_iterator __pos,
const key_type& __k);
private:
template<typename _Arg, typename _NodeGen>
iterator
_M_insert_(_Base_ptr __x, _Base_ptr __y, _Arg&& __v, _NodeGen&);
iterator
_M_insert_node(_Base_ptr __x, _Base_ptr __y, _Link_type __z);
template<typename _Arg>
iterator
_M_insert_lower(_Base_ptr __y, _Arg&& __v);
template<typename _Arg>
iterator
_M_insert_equal_lower(_Arg&& __x);
iterator
_M_insert_lower_node(_Base_ptr __p, _Link_type __z);
iterator
_M_insert_equal_lower_node(_Link_type __z);
enum { __as_lvalue, __as_rvalue };
template<bool _MoveValues, typename _NodeGen>
_Link_type
_M_copy(_Link_type, _Base_ptr, _NodeGen&);
template<bool _MoveValues, typename _NodeGen>
_Link_type
_M_copy(const _Rb_tree& __x, _NodeGen& __gen)
{
_Link_type __root =
_M_copy<_MoveValues>(__x._M_mbegin(), _M_end(), __gen);
_M_leftmost() = _S_minimum(__root);
_M_rightmost() = _S_maximum(__root);
_M_impl._M_node_count = __x._M_impl._M_node_count;
return __root;
}
_Link_type
_M_copy(const _Rb_tree& __x)
{
_Alloc_node __an(*this);
return _M_copy<__as_lvalue>(__x, __an);
}
void
_M_erase(_Link_type __x);
iterator
_M_lower_bound(_Link_type __x, _Base_ptr __y,
const _Key& __k);
const_iterator
_M_lower_bound(_Const_Link_type __x, _Const_Base_ptr __y,
const _Key& __k) const;
iterator
_M_upper_bound(_Link_type __x, _Base_ptr __y,
const _Key& __k);
const_iterator
_M_upper_bound(_Const_Link_type __x, _Const_Base_ptr __y,
const _Key& __k) const;
public:
_Rb_tree() = default;
_Rb_tree(const _Compare& __comp,
const allocator_type& __a = allocator_type())
: _M_impl(__comp, _Node_allocator(__a)) { }
_Rb_tree(const _Rb_tree& __x)
: _M_impl(__x._M_impl)
{
if (__x._M_root() != 0)
_M_root() = _M_copy(__x);
}
_Rb_tree(const allocator_type& __a)
: _M_impl(_Node_allocator(__a))
{ }
_Rb_tree(const _Rb_tree& __x, const allocator_type& __a)
: _M_impl(__x._M_impl._M_key_compare, _Node_allocator(__a))
{
if (__x._M_root() != nullptr)
_M_root() = _M_copy(__x);
}
_Rb_tree(_Rb_tree&&) = default;
_Rb_tree(_Rb_tree&& __x, const allocator_type& __a)
: _Rb_tree(std::move(__x), _Node_allocator(__a))
{ }
private:
_Rb_tree(_Rb_tree&& __x, _Node_allocator&& __a, true_type)
noexcept(is_nothrow_default_constructible<_Compare>::value)
: _M_impl(std::move(__x._M_impl), std::move(__a))
{ }
_Rb_tree(_Rb_tree&& __x, _Node_allocator&& __a, false_type)
: _M_impl(__x._M_impl._M_key_compare, std::move(__a))
{
if (__x._M_root() != nullptr)
_M_move_data(__x, false_type{});
}
public:
_Rb_tree(_Rb_tree&& __x, _Node_allocator&& __a)
noexcept( noexcept(
_Rb_tree(std::declval<_Rb_tree&&>(), std::declval<_Node_allocator&&>(),
std::declval<typename _Alloc_traits::is_always_equal>())) )
: _Rb_tree(std::move(__x), std::move(__a),
typename _Alloc_traits::is_always_equal{})
{ }
~_Rb_tree() noexcept
{ _M_erase(_M_begin()); }
_Rb_tree&
operator=(const _Rb_tree& __x);
_Compare
key_comp() const
{ return _M_impl._M_key_compare; }
iterator
begin() noexcept
{ return iterator(this->_M_impl._M_header._M_left); }
const_iterator
begin() const noexcept
{ return const_iterator(this->_M_impl._M_header._M_left); }
iterator
end() noexcept
{ return iterator(&this->_M_impl._M_header); }
const_iterator
end() const noexcept
{ return const_iterator(&this->_M_impl._M_header); }
reverse_iterator
rbegin() noexcept
{ return reverse_iterator(end()); }
const_reverse_iterator
rbegin() const noexcept
{ return const_reverse_iterator(end()); }
reverse_iterator
rend() noexcept
{ return reverse_iterator(begin()); }
const_reverse_iterator
rend() const noexcept
{ return const_reverse_iterator(begin()); }
[[__nodiscard__]] bool
empty() const noexcept
{ return _M_impl._M_node_count == 0; }
size_type
size() const noexcept
{ return _M_impl._M_node_count; }
size_type
max_size() const noexcept
{ return _Alloc_traits::max_size(_M_get_Node_allocator()); }
void
swap(_Rb_tree& __t)
noexcept(__is_nothrow_swappable<_Compare>::value);
template<typename _Arg>
pair<iterator, bool>
_M_insert_unique(_Arg&& __x);
template<typename _Arg>
iterator
_M_insert_equal(_Arg&& __x);
template<typename _Arg, typename _NodeGen>
iterator
_M_insert_unique_(const_iterator __pos, _Arg&& __x, _NodeGen&);
template<typename _Arg>
iterator
_M_insert_unique_(const_iterator __pos, _Arg&& __x)
{
_Alloc_node __an(*this);
return _M_insert_unique_(__pos, std::forward<_Arg>(__x), __an);
}
template<typename _Arg, typename _NodeGen>
iterator
_M_insert_equal_(const_iterator __pos, _Arg&& __x, _NodeGen&);
template<typename _Arg>
iterator
_M_insert_equal_(const_iterator __pos, _Arg&& __x)
{
_Alloc_node __an(*this);
return _M_insert_equal_(__pos, std::forward<_Arg>(__x), __an);
}
template<typename... _Args>
pair<iterator, bool>
_M_emplace_unique(_Args&&... __args);
template<typename... _Args>
iterator
_M_emplace_equal(_Args&&... __args);
template<typename... _Args>
iterator
_M_emplace_hint_unique(const_iterator __pos, _Args&&... __args);
template<typename... _Args>
iterator
_M_emplace_hint_equal(const_iterator __pos, _Args&&... __args);
template<typename _Iter>
using __same_value_type
= is_same<value_type, typename iterator_traits<_Iter>::value_type>;
template<typename _InputIterator>
__enable_if_t<__same_value_type<_InputIterator>::value>
_M_insert_range_unique(_InputIterator __first, _InputIterator __last)
{
_Alloc_node __an(*this);
for (; __first != __last; ++__first)
_M_insert_unique_(end(), *__first, __an);
}
template<typename _InputIterator>
__enable_if_t<!__same_value_type<_InputIterator>::value>
_M_insert_range_unique(_InputIterator __first, _InputIterator __last)
{
for (; __first != __last; ++__first)
_M_emplace_unique(*__first);
}
template<typename _InputIterator>
__enable_if_t<__same_value_type<_InputIterator>::value>
_M_insert_range_equal(_InputIterator __first, _InputIterator __last)
{
_Alloc_node __an(*this);
for (; __first != __last; ++__first)
_M_insert_equal_(end(), *__first, __an);
}
template<typename _InputIterator>
__enable_if_t<!__same_value_type<_InputIterator>::value>
_M_insert_range_equal(_InputIterator __first, _InputIterator __last)
{
for (; __first != __last; ++__first)
_M_emplace_equal(*__first);
}
private:
void
_M_erase_aux(const_iterator __position);
void
_M_erase_aux(const_iterator __first, const_iterator __last);
public:
__attribute ((__abi_tag__ ("cxx11")))
iterator
erase(const_iterator __position)
{
do { if (std::__is_constant_evaluated() && !bool(__position != end())) __builtin_unreachable(); } while (false);
const_iterator __result = __position;
++__result;
_M_erase_aux(__position);
return __result._M_const_cast();
}
__attribute ((__abi_tag__ ("cxx11")))
iterator
erase(iterator __position)
{
do { if (std::__is_constant_evaluated() && !bool(__position != end())) __builtin_unreachable(); } while (false);
iterator __result = __position;
++__result;
_M_erase_aux(__position);
return __result;
}
size_type
erase(const key_type& __x);
__attribute ((__abi_tag__ ("cxx11")))
iterator
erase(const_iterator __first, const_iterator __last)
{
_M_erase_aux(__first, __last);
return __last._M_const_cast();
}
void
clear() noexcept
{
_M_erase(_M_begin());
_M_impl._M_reset();
}
iterator
find(const key_type& __k);
const_iterator
find(const key_type& __k) const;
size_type
count(const key_type& __k) const;
iterator
lower_bound(const key_type& __k)
{ return _M_lower_bound(_M_begin(), _M_end(), __k); }
const_iterator
lower_bound(const key_type& __k) const
{ return _M_lower_bound(_M_begin(), _M_end(), __k); }
iterator
upper_bound(const key_type& __k)
{ return _M_upper_bound(_M_begin(), _M_end(), __k); }
const_iterator
upper_bound(const key_type& __k) const
{ return _M_upper_bound(_M_begin(), _M_end(), __k); }
pair<iterator, iterator>
equal_range(const key_type& __k);
pair<const_iterator, const_iterator>
equal_range(const key_type& __k) const;
template<typename _Kt,
typename _Req = __has_is_transparent_t<_Compare, _Kt>>
iterator
_M_find_tr(const _Kt& __k)
{
const _Rb_tree* __const_this = this;
return __const_this->_M_find_tr(__k)._M_const_cast();
}
template<typename _Kt,
typename _Req = __has_is_transparent_t<_Compare, _Kt>>
const_iterator
_M_find_tr(const _Kt& __k) const
{
auto __j = _M_lower_bound_tr(__k);
if (__j != end() && _M_impl._M_key_compare(__k, _S_key(__j._M_node)))
__j = end();
return __j;
}
template<typename _Kt,
typename _Req = __has_is_transparent_t<_Compare, _Kt>>
size_type
_M_count_tr(const _Kt& __k) const
{
auto __p = _M_equal_range_tr(__k);
return std::distance(__p.first, __p.second);
}
template<typename _Kt,
typename _Req = __has_is_transparent_t<_Compare, _Kt>>
iterator
_M_lower_bound_tr(const _Kt& __k)
{
const _Rb_tree* __const_this = this;
return __const_this->_M_lower_bound_tr(__k)._M_const_cast();
}
template<typename _Kt,
typename _Req = __has_is_transparent_t<_Compare, _Kt>>
const_iterator
_M_lower_bound_tr(const _Kt& __k) const
{
auto __x = _M_begin();
auto __y = _M_end();
while (__x != 0)
if (!_M_impl._M_key_compare(_S_key(__x), __k))
{
__y = __x;
__x = _S_left(__x);
}
else
__x = _S_right(__x);
return const_iterator(__y);
}
template<typename _Kt,
typename _Req = __has_is_transparent_t<_Compare, _Kt>>
iterator
_M_upper_bound_tr(const _Kt& __k)
{
const _Rb_tree* __const_this = this;
return __const_this->_M_upper_bound_tr(__k)._M_const_cast();
}
template<typename _Kt,
typename _Req = __has_is_transparent_t<_Compare, _Kt>>
const_iterator
_M_upper_bound_tr(const _Kt& __k) const
{
auto __x = _M_begin();
auto __y = _M_end();
while (__x != 0)
if (_M_impl._M_key_compare(__k, _S_key(__x)))
{
__y = __x;
__x = _S_left(__x);
}
else
__x = _S_right(__x);
return const_iterator(__y);
}
template<typename _Kt,
typename _Req = __has_is_transparent_t<_Compare, _Kt>>
pair<iterator, iterator>
_M_equal_range_tr(const _Kt& __k)
{
const _Rb_tree* __const_this = this;
auto __ret = __const_this->_M_equal_range_tr(__k);
return { __ret.first._M_const_cast(), __ret.second._M_const_cast() };
}
template<typename _Kt,
typename _Req = __has_is_transparent_t<_Compare, _Kt>>
pair<const_iterator, const_iterator>
_M_equal_range_tr(const _Kt& __k) const
{
auto __low = _M_lower_bound_tr(__k);
auto __high = __low;
auto& __cmp = _M_impl._M_key_compare;
while (__high != end() && !__cmp(__k, _S_key(__high._M_node)))
++__high;
return { __low, __high };
}
bool
__rb_verify() const;
_Rb_tree&
operator=(_Rb_tree&&)
noexcept(_Alloc_traits::_S_nothrow_move()
&& is_nothrow_move_assignable<_Compare>::value);
template<typename _Iterator>
void
_M_assign_unique(_Iterator, _Iterator);
template<typename _Iterator>
void
_M_assign_equal(_Iterator, _Iterator);
private:
void
_M_move_data(_Rb_tree& __x, true_type)
{ _M_impl._M_move_data(__x._M_impl); }
void
_M_move_data(_Rb_tree&, false_type);
void
_M_move_assign(_Rb_tree&, true_type);
void
_M_move_assign(_Rb_tree&, false_type);
public:
insert_return_type
_M_reinsert_node_unique(node_type&& __nh)
{
insert_return_type __ret;
if (__nh.empty())
__ret.position = end();
else
{
do { if (std::__is_constant_evaluated() && !bool(_M_get_Node_allocator() == *__nh._M_alloc)) __builtin_unreachable(); } while (false);
auto __res = _M_get_insert_unique_pos(__nh._M_key());
if (__res.second)
{
__ret.position
= _M_insert_node(__res.first, __res.second, __nh._M_ptr);
__nh._M_ptr = nullptr;
__ret.inserted = true;
}
else
{
__ret.node = std::move(__nh);
__ret.position = iterator(__res.first);
__ret.inserted = false;
}
}
return __ret;
}
iterator
_M_reinsert_node_equal(node_type&& __nh)
{
iterator __ret;
if (__nh.empty())
__ret = end();
else
{
do { if (std::__is_constant_evaluated() && !bool(_M_get_Node_allocator() == *__nh._M_alloc)) __builtin_unreachable(); } while (false);
auto __res = _M_get_insert_equal_pos(__nh._M_key());
if (__res.second)
__ret = _M_insert_node(__res.first, __res.second, __nh._M_ptr);
else
__ret = _M_insert_equal_lower_node(__nh._M_ptr);
__nh._M_ptr = nullptr;
}
return __ret;
}
iterator
_M_reinsert_node_hint_unique(const_iterator __hint, node_type&& __nh)
{
iterator __ret;
if (__nh.empty())
__ret = end();
else
{
do { if (std::__is_constant_evaluated() && !bool(_M_get_Node_allocator() == *__nh._M_alloc)) __builtin_unreachable(); } while (false);
auto __res = _M_get_insert_hint_unique_pos(__hint, __nh._M_key());
if (__res.second)
{
__ret = _M_insert_node(__res.first, __res.second, __nh._M_ptr);
__nh._M_ptr = nullptr;
}
else
__ret = iterator(__res.first);
}
return __ret;
}
iterator
_M_reinsert_node_hint_equal(const_iterator __hint, node_type&& __nh)
{
iterator __ret;
if (__nh.empty())
__ret = end();
else
{
do { if (std::__is_constant_evaluated() && !bool(_M_get_Node_allocator() == *__nh._M_alloc)) __builtin_unreachable(); } while (false);
auto __res = _M_get_insert_hint_equal_pos(__hint, __nh._M_key());
if (__res.second)
__ret = _M_insert_node(__res.first, __res.second, __nh._M_ptr);
else
__ret = _M_insert_equal_lower_node(__nh._M_ptr);
__nh._M_ptr = nullptr;
}
return __ret;
}
node_type
extract(const_iterator __pos)
{
auto __ptr = _Rb_tree_rebalance_for_erase(
__pos._M_const_cast()._M_node, _M_impl._M_header);
--_M_impl._M_node_count;
return { static_cast<_Link_type>(__ptr), _M_get_Node_allocator() };
}
node_type
extract(const key_type& __k)
{
node_type __nh;
auto __pos = find(__k);
if (__pos != end())
__nh = extract(const_iterator(__pos));
return __nh;
}
template<typename _Compare2>
using _Compatible_tree
= _Rb_tree<_Key, _Val, _KeyOfValue, _Compare2, _Alloc>;
template<typename, typename>
friend class _Rb_tree_merge_helper;
template<typename _Compare2>
void
_M_merge_unique(_Compatible_tree<_Compare2>& __src) noexcept
{
using _Merge_helper = _Rb_tree_merge_helper<_Rb_tree, _Compare2>;
for (auto __i = __src.begin(), __end = __src.end(); __i != __end;)
{
auto __pos = __i++;
auto __res = _M_get_insert_unique_pos(_KeyOfValue()(*__pos));
if (__res.second)
{
auto& __src_impl = _Merge_helper::_S_get_impl(__src);
auto __ptr = _Rb_tree_rebalance_for_erase(
__pos._M_node, __src_impl._M_header);
--__src_impl._M_node_count;
_M_insert_node(__res.first, __res.second,
static_cast<_Link_type>(__ptr));
}
}
}
template<typename _Compare2>
void
_M_merge_equal(_Compatible_tree<_Compare2>& __src) noexcept
{
using _Merge_helper = _Rb_tree_merge_helper<_Rb_tree, _Compare2>;
for (auto __i = __src.begin(), __end = __src.end(); __i != __end;)
{
auto __pos = __i++;
auto __res = _M_get_insert_equal_pos(_KeyOfValue()(*__pos));
if (__res.second)
{
auto& __src_impl = _Merge_helper::_S_get_impl(__src);
auto __ptr = _Rb_tree_rebalance_for_erase(
__pos._M_node, __src_impl._M_header);
--__src_impl._M_node_count;
_M_insert_node(__res.first, __res.second,
static_cast<_Link_type>(__ptr));
}
}
}
friend bool
operator==(const _Rb_tree& __x, const _Rb_tree& __y)
{
return __x.size() == __y.size()
&& std::equal(__x.begin(), __x.end(), __y.begin());
}
friend bool
operator<(const _Rb_tree& __x, const _Rb_tree& __y)
{
return std::lexicographical_compare(__x.begin(), __x.end(),
__y.begin(), __y.end());
}
private:
struct _Auto_node
{
template<typename... _Args>
_Auto_node(_Rb_tree& __t, _Args&&... __args)
: _M_t(__t),
_M_node(__t._M_create_node(std::forward<_Args>(__args)...))
{ }
~_Auto_node()
{
if (_M_node)
_M_t._M_drop_node(_M_node);
}
_Auto_node(_Auto_node&& __n)
: _M_t(__n._M_t), _M_node(__n._M_node)
{ __n._M_node = nullptr; }
const _Key&
_M_key() const
{ return _S_key(_M_node); }
iterator
_M_insert(pair<_Base_ptr, _Base_ptr> __p)
{
auto __it = _M_t._M_insert_node(__p.first, __p.second, _M_node);
_M_node = nullptr;
return __it;
}
iterator
_M_insert_equal_lower()
{
auto __it = _M_t._M_insert_equal_lower_node(_M_node);
_M_node = nullptr;
return __it;
}
_Rb_tree& _M_t;
_Link_type _M_node;
};
};
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
inline void
swap(_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>& __x,
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>& __y)
{ __x.swap(__y); }
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
void
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_move_data(_Rb_tree& __x, false_type)
{
if (_M_get_Node_allocator() == __x._M_get_Node_allocator())
_M_move_data(__x, true_type());
else
{
constexpr bool __move = !__move_if_noexcept_cond<value_type>::value;
_Alloc_node __an(*this);
_M_root() = _M_copy<__move>(__x, __an);
if constexpr (__move)
__x.clear();
}
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
inline void
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_move_assign(_Rb_tree& __x, true_type)
{
clear();
if (__x._M_root() != nullptr)
_M_move_data(__x, true_type());
std::__alloc_on_move(_M_get_Node_allocator(),
__x._M_get_Node_allocator());
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
void
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_move_assign(_Rb_tree& __x, false_type)
{
if (_M_get_Node_allocator() == __x._M_get_Node_allocator())
return _M_move_assign(__x, true_type{});
_Reuse_or_alloc_node __roan(*this);
_M_impl._M_reset();
if (__x._M_root() != nullptr)
{
_M_root() = _M_copy<__as_rvalue>(__x, __roan);
__x.clear();
}
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
inline _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>&
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
operator=(_Rb_tree&& __x)
noexcept(_Alloc_traits::_S_nothrow_move()
&& is_nothrow_move_assignable<_Compare>::value)
{
_M_impl._M_key_compare = std::move(__x._M_impl._M_key_compare);
_M_move_assign(__x, __bool_constant<_Alloc_traits::_S_nothrow_move()>());
return *this;
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
template<typename _Iterator>
void
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_assign_unique(_Iterator __first, _Iterator __last)
{
_Reuse_or_alloc_node __roan(*this);
_M_impl._M_reset();
for (; __first != __last; ++__first)
_M_insert_unique_(end(), *__first, __roan);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
template<typename _Iterator>
void
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_assign_equal(_Iterator __first, _Iterator __last)
{
_Reuse_or_alloc_node __roan(*this);
_M_impl._M_reset();
for (; __first != __last; ++__first)
_M_insert_equal_(end(), *__first, __roan);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>&
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
operator=(const _Rb_tree& __x)
{
if (this != std::__addressof(__x))
{
if (_Alloc_traits::_S_propagate_on_copy_assign())
{
auto& __this_alloc = this->_M_get_Node_allocator();
auto& __that_alloc = __x._M_get_Node_allocator();
if (!_Alloc_traits::_S_always_equal()
&& __this_alloc != __that_alloc)
{
clear();
std::__alloc_on_copy(__this_alloc, __that_alloc);
}
}
_Reuse_or_alloc_node __roan(*this);
_M_impl._M_reset();
_M_impl._M_key_compare = __x._M_impl._M_key_compare;
if (__x._M_root() != 0)
_M_root() = _M_copy<__as_lvalue>(__x, __roan);
}
return *this;
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
template<typename _Arg, typename _NodeGen>
typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_insert_(_Base_ptr __x, _Base_ptr __p,
_Arg&& __v,
_NodeGen& __node_gen)
{
bool __insert_left = (__x != 0 || __p == _M_end()
|| _M_impl._M_key_compare(_KeyOfValue()(__v),
_S_key(__p)));
_Link_type __z = __node_gen(std::forward<_Arg>(__v));
_Rb_tree_insert_and_rebalance(__insert_left, __z, __p,
this->_M_impl._M_header);
++_M_impl._M_node_count;
return iterator(__z);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
template<typename _Arg>
typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_insert_lower(_Base_ptr __p, _Arg&& __v)
{
bool __insert_left = (__p == _M_end()
|| !_M_impl._M_key_compare(_S_key(__p),
_KeyOfValue()(__v)));
_Link_type __z = _M_create_node(std::forward<_Arg>(__v));
_Rb_tree_insert_and_rebalance(__insert_left, __z, __p,
this->_M_impl._M_header);
++_M_impl._M_node_count;
return iterator(__z);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
template<typename _Arg>
typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_insert_equal_lower(_Arg&& __v)
{
_Link_type __x = _M_begin();
_Base_ptr __y = _M_end();
while (__x != 0)
{
__y = __x;
__x = !_M_impl._M_key_compare(_S_key(__x), _KeyOfValue()(__v)) ?
_S_left(__x) : _S_right(__x);
}
return _M_insert_lower(__y, std::forward<_Arg>(__v));
}
template<typename _Key, typename _Val, typename _KoV,
typename _Compare, typename _Alloc>
template<bool _MoveValues, typename _NodeGen>
typename _Rb_tree<_Key, _Val, _KoV, _Compare, _Alloc>::_Link_type
_Rb_tree<_Key, _Val, _KoV, _Compare, _Alloc>::
_M_copy(_Link_type __x, _Base_ptr __p, _NodeGen& __node_gen)
{
_Link_type __top = _M_clone_node<_MoveValues>(__x, __node_gen);
__top->_M_parent = __p;
try
{
if (__x->_M_right)
__top->_M_right =
_M_copy<_MoveValues>(_S_right(__x), __top, __node_gen);
__p = __top;
__x = _S_left(__x);
while (__x != 0)
{
_Link_type __y = _M_clone_node<_MoveValues>(__x, __node_gen);
__p->_M_left = __y;
__y->_M_parent = __p;
if (__x->_M_right)
__y->_M_right = _M_copy<_MoveValues>(_S_right(__x),
__y, __node_gen);
__p = __y;
__x = _S_left(__x);
}
}
catch(...)
{
_M_erase(__top);
throw;
}
return __top;
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
void
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_erase(_Link_type __x)
{
while (__x != 0)
{
_M_erase(_S_right(__x));
_Link_type __y = _S_left(__x);
_M_drop_node(__x);
__x = __y;
}
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::iterator
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_lower_bound(_Link_type __x, _Base_ptr __y,
const _Key& __k)
{
while (__x != 0)
if (!_M_impl._M_key_compare(_S_key(__x), __k))
__y = __x, __x = _S_left(__x);
else
__x = _S_right(__x);
return iterator(__y);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::const_iterator
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_lower_bound(_Const_Link_type __x, _Const_Base_ptr __y,
const _Key& __k) const
{
while (__x != 0)
if (!_M_impl._M_key_compare(_S_key(__x), __k))
__y = __x, __x = _S_left(__x);
else
__x = _S_right(__x);
return const_iterator(__y);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::iterator
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_upper_bound(_Link_type __x, _Base_ptr __y,
const _Key& __k)
{
while (__x != 0)
if (_M_impl._M_key_compare(__k, _S_key(__x)))
__y = __x, __x = _S_left(__x);
else
__x = _S_right(__x);
return iterator(__y);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::const_iterator
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_upper_bound(_Const_Link_type __x, _Const_Base_ptr __y,
const _Key& __k) const
{
while (__x != 0)
if (_M_impl._M_key_compare(__k, _S_key(__x)))
__y = __x, __x = _S_left(__x);
else
__x = _S_right(__x);
return const_iterator(__y);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
pair<typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::iterator,
typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::iterator>
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
equal_range(const _Key& __k)
{
_Link_type __x = _M_begin();
_Base_ptr __y = _M_end();
while (__x != 0)
{
if (_M_impl._M_key_compare(_S_key(__x), __k))
__x = _S_right(__x);
else if (_M_impl._M_key_compare(__k, _S_key(__x)))
__y = __x, __x = _S_left(__x);
else
{
_Link_type __xu(__x);
_Base_ptr __yu(__y);
__y = __x, __x = _S_left(__x);
__xu = _S_right(__xu);
return pair<iterator,
iterator>(_M_lower_bound(__x, __y, __k),
_M_upper_bound(__xu, __yu, __k));
}
}
return pair<iterator, iterator>(iterator(__y),
iterator(__y));
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
pair<typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::const_iterator,
typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::const_iterator>
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
equal_range(const _Key& __k) const
{
_Const_Link_type __x = _M_begin();
_Const_Base_ptr __y = _M_end();
while (__x != 0)
{
if (_M_impl._M_key_compare(_S_key(__x), __k))
__x = _S_right(__x);
else if (_M_impl._M_key_compare(__k, _S_key(__x)))
__y = __x, __x = _S_left(__x);
else
{
_Const_Link_type __xu(__x);
_Const_Base_ptr __yu(__y);
__y = __x, __x = _S_left(__x);
__xu = _S_right(__xu);
return pair<const_iterator,
const_iterator>(_M_lower_bound(__x, __y, __k),
_M_upper_bound(__xu, __yu, __k));
}
}
return pair<const_iterator, const_iterator>(const_iterator(__y),
const_iterator(__y));
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
void
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
swap(_Rb_tree& __t)
noexcept(__is_nothrow_swappable<_Compare>::value)
{
if (_M_root() == 0)
{
if (__t._M_root() != 0)
_M_impl._M_move_data(__t._M_impl);
}
else if (__t._M_root() == 0)
__t._M_impl._M_move_data(_M_impl);
else
{
std::swap(_M_root(),__t._M_root());
std::swap(_M_leftmost(),__t._M_leftmost());
std::swap(_M_rightmost(),__t._M_rightmost());
_M_root()->_M_parent = _M_end();
__t._M_root()->_M_parent = __t._M_end();
std::swap(this->_M_impl._M_node_count, __t._M_impl._M_node_count);
}
std::swap(this->_M_impl._M_key_compare, __t._M_impl._M_key_compare);
_Alloc_traits::_S_on_swap(_M_get_Node_allocator(),
__t._M_get_Node_allocator());
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
pair<typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::_Base_ptr,
typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::_Base_ptr>
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_get_insert_unique_pos(const key_type& __k)
{
typedef pair<_Base_ptr, _Base_ptr> _Res;
_Link_type __x = _M_begin();
_Base_ptr __y = _M_end();
bool __comp = true;
while (__x != 0)
{
__y = __x;
__comp = _M_impl._M_key_compare(__k, _S_key(__x));
__x = __comp ? _S_left(__x) : _S_right(__x);
}
iterator __j = iterator(__y);
if (__comp)
{
if (__j == begin())
return _Res(__x, __y);
else
--__j;
}
if (_M_impl._M_key_compare(_S_key(__j._M_node), __k))
return _Res(__x, __y);
return _Res(__j._M_node, 0);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
pair<typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::_Base_ptr,
typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::_Base_ptr>
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_get_insert_equal_pos(const key_type& __k)
{
typedef pair<_Base_ptr, _Base_ptr> _Res;
_Link_type __x = _M_begin();
_Base_ptr __y = _M_end();
while (__x != 0)
{
__y = __x;
__x = _M_impl._M_key_compare(__k, _S_key(__x)) ?
_S_left(__x) : _S_right(__x);
}
return _Res(__x, __y);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
template<typename _Arg>
pair<typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::iterator, bool>
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_insert_unique(_Arg&& __v)
{
typedef pair<iterator, bool> _Res;
pair<_Base_ptr, _Base_ptr> __res
= _M_get_insert_unique_pos(_KeyOfValue()(__v));
if (__res.second)
{
_Alloc_node __an(*this);
return _Res(_M_insert_(__res.first, __res.second,
std::forward<_Arg>(__v), __an),
true);
}
return _Res(iterator(__res.first), false);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
template<typename _Arg>
typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_insert_equal(_Arg&& __v)
{
pair<_Base_ptr, _Base_ptr> __res
= _M_get_insert_equal_pos(_KeyOfValue()(__v));
_Alloc_node __an(*this);
return _M_insert_(__res.first, __res.second,
std::forward<_Arg>(__v), __an);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
pair<typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::_Base_ptr,
typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::_Base_ptr>
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_get_insert_hint_unique_pos(const_iterator __position,
const key_type& __k)
{
iterator __pos = __position._M_const_cast();
typedef pair<_Base_ptr, _Base_ptr> _Res;
if (__pos._M_node == _M_end())
{
if (size() > 0
&& _M_impl._M_key_compare(_S_key(_M_rightmost()), __k))
return _Res(0, _M_rightmost());
else
return _M_get_insert_unique_pos(__k);
}
else if (_M_impl._M_key_compare(__k, _S_key(__pos._M_node)))
{
iterator __before = __pos;
if (__pos._M_node == _M_leftmost())
return _Res(_M_leftmost(), _M_leftmost());
else if (_M_impl._M_key_compare(_S_key((--__before)._M_node), __k))
{
if (_S_right(__before._M_node) == 0)
return _Res(0, __before._M_node);
else
return _Res(__pos._M_node, __pos._M_node);
}
else
return _M_get_insert_unique_pos(__k);
}
else if (_M_impl._M_key_compare(_S_key(__pos._M_node), __k))
{
iterator __after = __pos;
if (__pos._M_node == _M_rightmost())
return _Res(0, _M_rightmost());
else if (_M_impl._M_key_compare(__k, _S_key((++__after)._M_node)))
{
if (_S_right(__pos._M_node) == 0)
return _Res(0, __pos._M_node);
else
return _Res(__after._M_node, __after._M_node);
}
else
return _M_get_insert_unique_pos(__k);
}
else
return _Res(__pos._M_node, 0);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
template<typename _Arg, typename _NodeGen>
typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_insert_unique_(const_iterator __position,
_Arg&& __v,
_NodeGen& __node_gen)
{
pair<_Base_ptr, _Base_ptr> __res
= _M_get_insert_hint_unique_pos(__position, _KeyOfValue()(__v));
if (__res.second)
return _M_insert_(__res.first, __res.second,
std::forward<_Arg>(__v),
__node_gen);
return iterator(__res.first);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
pair<typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::_Base_ptr,
typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::_Base_ptr>
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_get_insert_hint_equal_pos(const_iterator __position, const key_type& __k)
{
iterator __pos = __position._M_const_cast();
typedef pair<_Base_ptr, _Base_ptr> _Res;
if (__pos._M_node == _M_end())
{
if (size() > 0
&& !_M_impl._M_key_compare(__k, _S_key(_M_rightmost())))
return _Res(0, _M_rightmost());
else
return _M_get_insert_equal_pos(__k);
}
else if (!_M_impl._M_key_compare(_S_key(__pos._M_node), __k))
{
iterator __before = __pos;
if (__pos._M_node == _M_leftmost())
return _Res(_M_leftmost(), _M_leftmost());
else if (!_M_impl._M_key_compare(__k, _S_key((--__before)._M_node)))
{
if (_S_right(__before._M_node) == 0)
return _Res(0, __before._M_node);
else
return _Res(__pos._M_node, __pos._M_node);
}
else
return _M_get_insert_equal_pos(__k);
}
else
{
iterator __after = __pos;
if (__pos._M_node == _M_rightmost())
return _Res(0, _M_rightmost());
else if (!_M_impl._M_key_compare(_S_key((++__after)._M_node), __k))
{
if (_S_right(__pos._M_node) == 0)
return _Res(0, __pos._M_node);
else
return _Res(__after._M_node, __after._M_node);
}
else
return _Res(0, 0);
}
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
template<typename _Arg, typename _NodeGen>
typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_insert_equal_(const_iterator __position,
_Arg&& __v,
_NodeGen& __node_gen)
{
pair<_Base_ptr, _Base_ptr> __res
= _M_get_insert_hint_equal_pos(__position, _KeyOfValue()(__v));
if (__res.second)
return _M_insert_(__res.first, __res.second,
std::forward<_Arg>(__v),
__node_gen);
return _M_insert_equal_lower(std::forward<_Arg>(__v));
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
auto
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_insert_node(_Base_ptr __x, _Base_ptr __p, _Link_type __z)
-> iterator
{
bool __insert_left = (__x != 0 || __p == _M_end()
|| _M_impl._M_key_compare(_S_key(__z),
_S_key(__p)));
_Rb_tree_insert_and_rebalance(__insert_left, __z, __p,
this->_M_impl._M_header);
++_M_impl._M_node_count;
return iterator(__z);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
auto
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_insert_lower_node(_Base_ptr __p, _Link_type __z)
-> iterator
{
bool __insert_left = (__p == _M_end()
|| !_M_impl._M_key_compare(_S_key(__p),
_S_key(__z)));
_Rb_tree_insert_and_rebalance(__insert_left, __z, __p,
this->_M_impl._M_header);
++_M_impl._M_node_count;
return iterator(__z);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
auto
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_insert_equal_lower_node(_Link_type __z)
-> iterator
{
_Link_type __x = _M_begin();
_Base_ptr __y = _M_end();
while (__x != 0)
{
__y = __x;
__x = !_M_impl._M_key_compare(_S_key(__x), _S_key(__z)) ?
_S_left(__x) : _S_right(__x);
}
return _M_insert_lower_node(__y, __z);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
template<typename... _Args>
auto
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_emplace_unique(_Args&&... __args)
-> pair<iterator, bool>
{
_Auto_node __z(*this, std::forward<_Args>(__args)...);
auto __res = _M_get_insert_unique_pos(__z._M_key());
if (__res.second)
return {__z._M_insert(__res), true};
return {iterator(__res.first), false};
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
template<typename... _Args>
auto
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_emplace_equal(_Args&&... __args)
-> iterator
{
_Auto_node __z(*this, std::forward<_Args>(__args)...);
auto __res = _M_get_insert_equal_pos(__z._M_key());
return __z._M_insert(__res);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
template<typename... _Args>
auto
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_emplace_hint_unique(const_iterator __pos, _Args&&... __args)
-> iterator
{
_Auto_node __z(*this, std::forward<_Args>(__args)...);
auto __res = _M_get_insert_hint_unique_pos(__pos, __z._M_key());
if (__res.second)
return __z._M_insert(__res);
return iterator(__res.first);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
template<typename... _Args>
auto
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_emplace_hint_equal(const_iterator __pos, _Args&&... __args)
-> iterator
{
_Auto_node __z(*this, std::forward<_Args>(__args)...);
auto __res = _M_get_insert_hint_equal_pos(__pos, __z._M_key());
if (__res.second)
return __z._M_insert(__res);
return __z._M_insert_equal_lower();
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
void
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_erase_aux(const_iterator __position)
{
_Link_type __y =
static_cast<_Link_type>(_Rb_tree_rebalance_for_erase
(const_cast<_Base_ptr>(__position._M_node),
this->_M_impl._M_header));
_M_drop_node(__y);
--_M_impl._M_node_count;
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
void
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
_M_erase_aux(const_iterator __first, const_iterator __last)
{
if (__first == begin() && __last == end())
clear();
else
while (__first != __last)
_M_erase_aux(__first++);
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::size_type
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
erase(const _Key& __x)
{
pair<iterator, iterator> __p = equal_range(__x);
const size_type __old_size = size();
_M_erase_aux(__p.first, __p.second);
return __old_size - size();
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::iterator
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
find(const _Key& __k)
{
iterator __j = _M_lower_bound(_M_begin(), _M_end(), __k);
return (__j == end()
|| _M_impl._M_key_compare(__k,
_S_key(__j._M_node))) ? end() : __j;
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
typename _Rb_tree<_Key, _Val, _KeyOfValue,
_Compare, _Alloc>::const_iterator
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
find(const _Key& __k) const
{
const_iterator __j = _M_lower_bound(_M_begin(), _M_end(), __k);
return (__j == end()
|| _M_impl._M_key_compare(__k,
_S_key(__j._M_node))) ? end() : __j;
}
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::size_type
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
count(const _Key& __k) const
{
pair<const_iterator, const_iterator> __p = equal_range(__k);
const size_type __n = std::distance(__p.first, __p.second);
return __n;
}
__attribute__ ((__pure__)) unsigned int
_Rb_tree_black_count(const _Rb_tree_node_base* __node,
const _Rb_tree_node_base* __root) throw ();
template<typename _Key, typename _Val, typename _KeyOfValue,
typename _Compare, typename _Alloc>
bool
_Rb_tree<_Key,_Val,_KeyOfValue,_Compare,_Alloc>::__rb_verify() const
{
if (_M_impl._M_node_count == 0 || begin() == end())
return _M_impl._M_node_count == 0 && begin() == end()
&& this->_M_impl._M_header._M_left == _M_end()
&& this->_M_impl._M_header._M_right == _M_end();
unsigned int __len = _Rb_tree_black_count(_M_leftmost(), _M_root());
for (const_iterator __it = begin(); __it != end(); ++__it)
{
_Const_Link_type __x = static_cast<_Const_Link_type>(__it._M_node);
_Const_Link_type __L = _S_left(__x);
_Const_Link_type __R = _S_right(__x);
if (__x->_M_color == _S_red)
if ((__L && __L->_M_color == _S_red)
|| (__R && __R->_M_color == _S_red))
return false;
if (__L && _M_impl._M_key_compare(_S_key(__x), _S_key(__L)))
return false;
if (__R && _M_impl._M_key_compare(_S_key(__R), _S_key(__x)))
return false;
if (!__L && !__R && _Rb_tree_black_count(__x, _M_root()) != __len)
return false;
}
if (_M_leftmost() != _Rb_tree_node_base::_S_minimum(_M_root()))
return false;
if (_M_rightmost() != _Rb_tree_node_base::_S_maximum(_M_root()))
return false;
return true;
}
template<typename _Key, typename _Val, typename _Sel, typename _Cmp1,
typename _Alloc, typename _Cmp2>
struct _Rb_tree_merge_helper<_Rb_tree<_Key, _Val, _Sel, _Cmp1, _Alloc>,
_Cmp2>
{
private:
friend class _Rb_tree<_Key, _Val, _Sel, _Cmp1, _Alloc>;
static auto&
_S_get_impl(_Rb_tree<_Key, _Val, _Sel, _Cmp2, _Alloc>& __tree)
{ return __tree._M_impl; }
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Key, typename _Compare, typename _Alloc>
class multiset;
template<typename _Key, typename _Compare = std::less<_Key>,
typename _Alloc = std::allocator<_Key> >
class set
{
static_assert(is_same<typename remove_cv<_Key>::type, _Key>::value,
"std::set must have a non-const, non-volatile value_type");
public:
typedef _Key key_type;
typedef _Key value_type;
typedef _Compare key_compare;
typedef _Compare value_compare;
typedef _Alloc allocator_type;
private:
typedef typename __gnu_cxx::__alloc_traits<_Alloc>::template
rebind<_Key>::other _Key_alloc_type;
typedef _Rb_tree<key_type, value_type, _Identity<value_type>,
key_compare, _Key_alloc_type> _Rep_type;
_Rep_type _M_t;
typedef __gnu_cxx::__alloc_traits<_Key_alloc_type> _Alloc_traits;
public:
typedef typename _Alloc_traits::pointer pointer;
typedef typename _Alloc_traits::const_pointer const_pointer;
typedef typename _Alloc_traits::reference reference;
typedef typename _Alloc_traits::const_reference const_reference;
typedef typename _Rep_type::const_iterator iterator;
typedef typename _Rep_type::const_iterator const_iterator;
typedef typename _Rep_type::const_reverse_iterator reverse_iterator;
typedef typename _Rep_type::const_reverse_iterator const_reverse_iterator;
typedef typename _Rep_type::size_type size_type;
typedef typename _Rep_type::difference_type difference_type;
using node_type = typename _Rep_type::node_type;
using insert_return_type = typename _Rep_type::insert_return_type;
set() = default;
explicit
set(const _Compare& __comp,
const allocator_type& __a = allocator_type())
: _M_t(__comp, _Key_alloc_type(__a)) { }
template<typename _InputIterator>
set(_InputIterator __first, _InputIterator __last)
: _M_t()
{ _M_t._M_insert_range_unique(__first, __last); }
template<typename _InputIterator>
set(_InputIterator __first, _InputIterator __last,
const _Compare& __comp,
const allocator_type& __a = allocator_type())
: _M_t(__comp, _Key_alloc_type(__a))
{ _M_t._M_insert_range_unique(__first, __last); }
set(const set&) = default;
set(set&&) = default;
set(initializer_list<value_type> __l,
const _Compare& __comp = _Compare(),
const allocator_type& __a = allocator_type())
: _M_t(__comp, _Key_alloc_type(__a))
{ _M_t._M_insert_range_unique(__l.begin(), __l.end()); }
explicit
set(const allocator_type& __a)
: _M_t(_Key_alloc_type(__a)) { }
set(const set& __x, const __type_identity_t<allocator_type>& __a)
: _M_t(__x._M_t, _Key_alloc_type(__a)) { }
set(set&& __x, const __type_identity_t<allocator_type>& __a)
noexcept(is_nothrow_copy_constructible<_Compare>::value
&& _Alloc_traits::_S_always_equal())
: _M_t(std::move(__x._M_t), _Key_alloc_type(__a)) { }
set(initializer_list<value_type> __l, const allocator_type& __a)
: _M_t(_Key_alloc_type(__a))
{ _M_t._M_insert_range_unique(__l.begin(), __l.end()); }
template<typename _InputIterator>
set(_InputIterator __first, _InputIterator __last,
const allocator_type& __a)
: _M_t(_Key_alloc_type(__a))
{ _M_t._M_insert_range_unique(__first, __last); }
~set() = default;
set&
operator=(const set&) = default;
set&
operator=(set&&) = default;
set&
operator=(initializer_list<value_type> __l)
{
_M_t._M_assign_unique(__l.begin(), __l.end());
return *this;
}
key_compare
key_comp() const
{ return _M_t.key_comp(); }
value_compare
value_comp() const
{ return _M_t.key_comp(); }
allocator_type
get_allocator() const noexcept
{ return allocator_type(_M_t.get_allocator()); }
iterator
begin() const noexcept
{ return _M_t.begin(); }
iterator
end() const noexcept
{ return _M_t.end(); }
reverse_iterator
rbegin() const noexcept
{ return _M_t.rbegin(); }
reverse_iterator
rend() const noexcept
{ return _M_t.rend(); }
iterator
cbegin() const noexcept
{ return _M_t.begin(); }
iterator
cend() const noexcept
{ return _M_t.end(); }
reverse_iterator
crbegin() const noexcept
{ return _M_t.rbegin(); }
reverse_iterator
crend() const noexcept
{ return _M_t.rend(); }
[[__nodiscard__]] bool
empty() const noexcept
{ return _M_t.empty(); }
size_type
size() const noexcept
{ return _M_t.size(); }
size_type
max_size() const noexcept
{ return _M_t.max_size(); }
void
swap(set& __x)
noexcept(__is_nothrow_swappable<_Compare>::value)
{ _M_t.swap(__x._M_t); }
template<typename... _Args>
std::pair<iterator, bool>
emplace(_Args&&... __args)
{ return _M_t._M_emplace_unique(std::forward<_Args>(__args)...); }
template<typename... _Args>
iterator
emplace_hint(const_iterator __pos, _Args&&... __args)
{
return _M_t._M_emplace_hint_unique(__pos,
std::forward<_Args>(__args)...);
}
std::pair<iterator, bool>
insert(const value_type& __x)
{
std::pair<typename _Rep_type::iterator, bool> __p =
_M_t._M_insert_unique(__x);
return std::pair<iterator, bool>(__p.first, __p.second);
}
std::pair<iterator, bool>
insert(value_type&& __x)
{
std::pair<typename _Rep_type::iterator, bool> __p =
_M_t._M_insert_unique(std::move(__x));
return std::pair<iterator, bool>(__p.first, __p.second);
}
iterator
insert(const_iterator __position, const value_type& __x)
{ return _M_t._M_insert_unique_(__position, __x); }
iterator
insert(const_iterator __position, value_type&& __x)
{ return _M_t._M_insert_unique_(__position, std::move(__x)); }
template<typename _InputIterator>
void
insert(_InputIterator __first, _InputIterator __last)
{ _M_t._M_insert_range_unique(__first, __last); }
void
insert(initializer_list<value_type> __l)
{ this->insert(__l.begin(), __l.end()); }
node_type
extract(const_iterator __pos)
{
do { if (std::__is_constant_evaluated() && !bool(__pos != end())) __builtin_unreachable(); } while (false);
return _M_t.extract(__pos);
}
node_type
extract(const key_type& __x)
{ return _M_t.extract(__x); }
insert_return_type
insert(node_type&& __nh)
{ return _M_t._M_reinsert_node_unique(std::move(__nh)); }
iterator
insert(const_iterator __hint, node_type&& __nh)
{ return _M_t._M_reinsert_node_hint_unique(__hint, std::move(__nh)); }
template<typename, typename>
friend struct std::_Rb_tree_merge_helper;
template<typename _Compare1>
void
merge(set<_Key, _Compare1, _Alloc>& __source)
{
using _Merge_helper = _Rb_tree_merge_helper<set, _Compare1>;
_M_t._M_merge_unique(_Merge_helper::_S_get_tree(__source));
}
template<typename _Compare1>
void
merge(set<_Key, _Compare1, _Alloc>&& __source)
{ merge(__source); }
template<typename _Compare1>
void
merge(multiset<_Key, _Compare1, _Alloc>& __source)
{
using _Merge_helper = _Rb_tree_merge_helper<set, _Compare1>;
_M_t._M_merge_unique(_Merge_helper::_S_get_tree(__source));
}
template<typename _Compare1>
void
merge(multiset<_Key, _Compare1, _Alloc>&& __source)
{ merge(__source); }
__attribute ((__abi_tag__ ("cxx11")))
iterator
erase(const_iterator __position)
{ return _M_t.erase(__position); }
size_type
erase(const key_type& __x)
{ return _M_t.erase(__x); }
__attribute ((__abi_tag__ ("cxx11")))
iterator
erase(const_iterator __first, const_iterator __last)
{ return _M_t.erase(__first, __last); }
void
clear() noexcept
{ _M_t.clear(); }
size_type
count(const key_type& __x) const
{ return _M_t.find(__x) == _M_t.end() ? 0 : 1; }
template<typename _Kt>
auto
count(const _Kt& __x) const
-> decltype(_M_t._M_count_tr(__x))
{ return _M_t._M_count_tr(__x); }
iterator
find(const key_type& __x)
{ return _M_t.find(__x); }
const_iterator
find(const key_type& __x) const
{ return _M_t.find(__x); }
template<typename _Kt>
auto
find(const _Kt& __x)
-> decltype(iterator{_M_t._M_find_tr(__x)})
{ return iterator{_M_t._M_find_tr(__x)}; }
template<typename _Kt>
auto
find(const _Kt& __x) const
-> decltype(const_iterator{_M_t._M_find_tr(__x)})
{ return const_iterator{_M_t._M_find_tr(__x)}; }
iterator
lower_bound(const key_type& __x)
{ return _M_t.lower_bound(__x); }
const_iterator
lower_bound(const key_type& __x) const
{ return _M_t.lower_bound(__x); }
template<typename _Kt>
auto
lower_bound(const _Kt& __x)
-> decltype(iterator(_M_t._M_lower_bound_tr(__x)))
{ return iterator(_M_t._M_lower_bound_tr(__x)); }
template<typename _Kt>
auto
lower_bound(const _Kt& __x) const
-> decltype(const_iterator(_M_t._M_lower_bound_tr(__x)))
{ return const_iterator(_M_t._M_lower_bound_tr(__x)); }
iterator
upper_bound(const key_type& __x)
{ return _M_t.upper_bound(__x); }
const_iterator
upper_bound(const key_type& __x) const
{ return _M_t.upper_bound(__x); }
template<typename _Kt>
auto
upper_bound(const _Kt& __x)
-> decltype(iterator(_M_t._M_upper_bound_tr(__x)))
{ return iterator(_M_t._M_upper_bound_tr(__x)); }
template<typename _Kt>
auto
upper_bound(const _Kt& __x) const
-> decltype(iterator(_M_t._M_upper_bound_tr(__x)))
{ return const_iterator(_M_t._M_upper_bound_tr(__x)); }
std::pair<iterator, iterator>
equal_range(const key_type& __x)
{ return _M_t.equal_range(__x); }
std::pair<const_iterator, const_iterator>
equal_range(const key_type& __x) const
{ return _M_t.equal_range(__x); }
template<typename _Kt>
auto
equal_range(const _Kt& __x)
-> decltype(pair<iterator, iterator>(_M_t._M_equal_range_tr(__x)))
{ return pair<iterator, iterator>(_M_t._M_equal_range_tr(__x)); }
template<typename _Kt>
auto
equal_range(const _Kt& __x) const
-> decltype(pair<iterator, iterator>(_M_t._M_equal_range_tr(__x)))
{ return pair<iterator, iterator>(_M_t._M_equal_range_tr(__x)); }
template<typename _K1, typename _C1, typename _A1>
friend bool
operator==(const set<_K1, _C1, _A1>&, const set<_K1, _C1, _A1>&);
template<typename _K1, typename _C1, typename _A1>
friend bool
operator<(const set<_K1, _C1, _A1>&, const set<_K1, _C1, _A1>&);
};
template<typename _InputIterator,
typename _Compare =
less<typename iterator_traits<_InputIterator>::value_type>,
typename _Allocator =
allocator<typename iterator_traits<_InputIterator>::value_type>,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireNotAllocator<_Compare>,
typename = _RequireAllocator<_Allocator>>
set(_InputIterator, _InputIterator,
_Compare = _Compare(), _Allocator = _Allocator())
-> set<typename iterator_traits<_InputIterator>::value_type,
_Compare, _Allocator>;
template<typename _Key, typename _Compare = less<_Key>,
typename _Allocator = allocator<_Key>,
typename = _RequireNotAllocator<_Compare>,
typename = _RequireAllocator<_Allocator>>
set(initializer_list<_Key>,
_Compare = _Compare(), _Allocator = _Allocator())
-> set<_Key, _Compare, _Allocator>;
template<typename _InputIterator, typename _Allocator,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireAllocator<_Allocator>>
set(_InputIterator, _InputIterator, _Allocator)
-> set<typename iterator_traits<_InputIterator>::value_type,
less<typename iterator_traits<_InputIterator>::value_type>,
_Allocator>;
template<typename _Key, typename _Allocator,
typename = _RequireAllocator<_Allocator>>
set(initializer_list<_Key>, _Allocator)
-> set<_Key, less<_Key>, _Allocator>;
template<typename _Key, typename _Compare, typename _Alloc>
inline bool
operator==(const set<_Key, _Compare, _Alloc>& __x,
const set<_Key, _Compare, _Alloc>& __y)
{ return __x._M_t == __y._M_t; }
template<typename _Key, typename _Compare, typename _Alloc>
inline bool
operator<(const set<_Key, _Compare, _Alloc>& __x,
const set<_Key, _Compare, _Alloc>& __y)
{ return __x._M_t < __y._M_t; }
template<typename _Key, typename _Compare, typename _Alloc>
inline bool
operator!=(const set<_Key, _Compare, _Alloc>& __x,
const set<_Key, _Compare, _Alloc>& __y)
{ return !(__x == __y); }
template<typename _Key, typename _Compare, typename _Alloc>
inline bool
operator>(const set<_Key, _Compare, _Alloc>& __x,
const set<_Key, _Compare, _Alloc>& __y)
{ return __y < __x; }
template<typename _Key, typename _Compare, typename _Alloc>
inline bool
operator<=(const set<_Key, _Compare, _Alloc>& __x,
const set<_Key, _Compare, _Alloc>& __y)
{ return !(__y < __x); }
template<typename _Key, typename _Compare, typename _Alloc>
inline bool
operator>=(const set<_Key, _Compare, _Alloc>& __x,
const set<_Key, _Compare, _Alloc>& __y)
{ return !(__x < __y); }
template<typename _Key, typename _Compare, typename _Alloc>
inline void
swap(set<_Key, _Compare, _Alloc>& __x, set<_Key, _Compare, _Alloc>& __y)
noexcept(noexcept(__x.swap(__y)))
{ __x.swap(__y); }
template<typename _Val, typename _Cmp1, typename _Alloc, typename _Cmp2>
struct
_Rb_tree_merge_helper<std::set<_Val, _Cmp1, _Alloc>, _Cmp2>
{
private:
friend class std::set<_Val, _Cmp1, _Alloc>;
static auto&
_S_get_tree(std::set<_Val, _Cmp2, _Alloc>& __set)
{ return __set._M_t; }
static auto&
_S_get_tree(std::multiset<_Val, _Cmp2, _Alloc>& __set)
{ return __set._M_t; }
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Key, typename _Compare, typename _Alloc>
class set;
template <typename _Key, typename _Compare = std::less<_Key>,
typename _Alloc = std::allocator<_Key> >
class multiset
{
static_assert(is_same<typename remove_cv<_Key>::type, _Key>::value,
"std::multiset must have a non-const, non-volatile value_type");
public:
typedef _Key key_type;
typedef _Key value_type;
typedef _Compare key_compare;
typedef _Compare value_compare;
typedef _Alloc allocator_type;
private:
typedef typename __gnu_cxx::__alloc_traits<_Alloc>::template
rebind<_Key>::other _Key_alloc_type;
typedef _Rb_tree<key_type, value_type, _Identity<value_type>,
key_compare, _Key_alloc_type> _Rep_type;
_Rep_type _M_t;
typedef __gnu_cxx::__alloc_traits<_Key_alloc_type> _Alloc_traits;
public:
typedef typename _Alloc_traits::pointer pointer;
typedef typename _Alloc_traits::const_pointer const_pointer;
typedef typename _Alloc_traits::reference reference;
typedef typename _Alloc_traits::const_reference const_reference;
typedef typename _Rep_type::const_iterator iterator;
typedef typename _Rep_type::const_iterator const_iterator;
typedef typename _Rep_type::const_reverse_iterator reverse_iterator;
typedef typename _Rep_type::const_reverse_iterator const_reverse_iterator;
typedef typename _Rep_type::size_type size_type;
typedef typename _Rep_type::difference_type difference_type;
using node_type = typename _Rep_type::node_type;
multiset() = default;
explicit
multiset(const _Compare& __comp,
const allocator_type& __a = allocator_type())
: _M_t(__comp, _Key_alloc_type(__a)) { }
template<typename _InputIterator>
multiset(_InputIterator __first, _InputIterator __last)
: _M_t()
{ _M_t._M_insert_range_equal(__first, __last); }
template<typename _InputIterator>
multiset(_InputIterator __first, _InputIterator __last,
const _Compare& __comp,
const allocator_type& __a = allocator_type())
: _M_t(__comp, _Key_alloc_type(__a))
{ _M_t._M_insert_range_equal(__first, __last); }
multiset(const multiset&) = default;
multiset(multiset&&) = default;
multiset(initializer_list<value_type> __l,
const _Compare& __comp = _Compare(),
const allocator_type& __a = allocator_type())
: _M_t(__comp, _Key_alloc_type(__a))
{ _M_t._M_insert_range_equal(__l.begin(), __l.end()); }
explicit
multiset(const allocator_type& __a)
: _M_t(_Key_alloc_type(__a)) { }
multiset(const multiset& __m,
const __type_identity_t<allocator_type>& __a)
: _M_t(__m._M_t, _Key_alloc_type(__a)) { }
multiset(multiset&& __m, const __type_identity_t<allocator_type>& __a)
noexcept(is_nothrow_copy_constructible<_Compare>::value
&& _Alloc_traits::_S_always_equal())
: _M_t(std::move(__m._M_t), _Key_alloc_type(__a)) { }
multiset(initializer_list<value_type> __l, const allocator_type& __a)
: _M_t(_Key_alloc_type(__a))
{ _M_t._M_insert_range_equal(__l.begin(), __l.end()); }
template<typename _InputIterator>
multiset(_InputIterator __first, _InputIterator __last,
const allocator_type& __a)
: _M_t(_Key_alloc_type(__a))
{ _M_t._M_insert_range_equal(__first, __last); }
~multiset() = default;
multiset&
operator=(const multiset&) = default;
multiset&
operator=(multiset&&) = default;
multiset&
operator=(initializer_list<value_type> __l)
{
_M_t._M_assign_equal(__l.begin(), __l.end());
return *this;
}
key_compare
key_comp() const
{ return _M_t.key_comp(); }
value_compare
value_comp() const
{ return _M_t.key_comp(); }
allocator_type
get_allocator() const noexcept
{ return allocator_type(_M_t.get_allocator()); }
iterator
begin() const noexcept
{ return _M_t.begin(); }
iterator
end() const noexcept
{ return _M_t.end(); }
reverse_iterator
rbegin() const noexcept
{ return _M_t.rbegin(); }
reverse_iterator
rend() const noexcept
{ return _M_t.rend(); }
iterator
cbegin() const noexcept
{ return _M_t.begin(); }
iterator
cend() const noexcept
{ return _M_t.end(); }
reverse_iterator
crbegin() const noexcept
{ return _M_t.rbegin(); }
reverse_iterator
crend() const noexcept
{ return _M_t.rend(); }
[[__nodiscard__]] bool
empty() const noexcept
{ return _M_t.empty(); }
size_type
size() const noexcept
{ return _M_t.size(); }
size_type
max_size() const noexcept
{ return _M_t.max_size(); }
void
swap(multiset& __x)
noexcept(__is_nothrow_swappable<_Compare>::value)
{ _M_t.swap(__x._M_t); }
template<typename... _Args>
iterator
emplace(_Args&&... __args)
{ return _M_t._M_emplace_equal(std::forward<_Args>(__args)...); }
template<typename... _Args>
iterator
emplace_hint(const_iterator __pos, _Args&&... __args)
{
return _M_t._M_emplace_hint_equal(__pos,
std::forward<_Args>(__args)...);
}
iterator
insert(const value_type& __x)
{ return _M_t._M_insert_equal(__x); }
iterator
insert(value_type&& __x)
{ return _M_t._M_insert_equal(std::move(__x)); }
iterator
insert(const_iterator __position, const value_type& __x)
{ return _M_t._M_insert_equal_(__position, __x); }
iterator
insert(const_iterator __position, value_type&& __x)
{ return _M_t._M_insert_equal_(__position, std::move(__x)); }
template<typename _InputIterator>
void
insert(_InputIterator __first, _InputIterator __last)
{ _M_t._M_insert_range_equal(__first, __last); }
void
insert(initializer_list<value_type> __l)
{ this->insert(__l.begin(), __l.end()); }
node_type
extract(const_iterator __pos)
{
do { if (std::__is_constant_evaluated() && !bool(__pos != end())) __builtin_unreachable(); } while (false);
return _M_t.extract(__pos);
}
node_type
extract(const key_type& __x)
{ return _M_t.extract(__x); }
iterator
insert(node_type&& __nh)
{ return _M_t._M_reinsert_node_equal(std::move(__nh)); }
iterator
insert(const_iterator __hint, node_type&& __nh)
{ return _M_t._M_reinsert_node_hint_equal(__hint, std::move(__nh)); }
template<typename, typename>
friend struct std::_Rb_tree_merge_helper;
template<typename _Compare1>
void
merge(multiset<_Key, _Compare1, _Alloc>& __source)
{
using _Merge_helper = _Rb_tree_merge_helper<multiset, _Compare1>;
_M_t._M_merge_equal(_Merge_helper::_S_get_tree(__source));
}
template<typename _Compare1>
void
merge(multiset<_Key, _Compare1, _Alloc>&& __source)
{ merge(__source); }
template<typename _Compare1>
void
merge(set<_Key, _Compare1, _Alloc>& __source)
{
using _Merge_helper = _Rb_tree_merge_helper<multiset, _Compare1>;
_M_t._M_merge_equal(_Merge_helper::_S_get_tree(__source));
}
template<typename _Compare1>
void
merge(set<_Key, _Compare1, _Alloc>&& __source)
{ merge(__source); }
__attribute ((__abi_tag__ ("cxx11")))
iterator
erase(const_iterator __position)
{ return _M_t.erase(__position); }
size_type
erase(const key_type& __x)
{ return _M_t.erase(__x); }
__attribute ((__abi_tag__ ("cxx11")))
iterator
erase(const_iterator __first, const_iterator __last)
{ return _M_t.erase(__first, __last); }
void
clear() noexcept
{ _M_t.clear(); }
size_type
count(const key_type& __x) const
{ return _M_t.count(__x); }
template<typename _Kt>
auto
count(const _Kt& __x) const -> decltype(_M_t._M_count_tr(__x))
{ return _M_t._M_count_tr(__x); }
iterator
find(const key_type& __x)
{ return _M_t.find(__x); }
const_iterator
find(const key_type& __x) const
{ return _M_t.find(__x); }
template<typename _Kt>
auto
find(const _Kt& __x)
-> decltype(iterator{_M_t._M_find_tr(__x)})
{ return iterator{_M_t._M_find_tr(__x)}; }
template<typename _Kt>
auto
find(const _Kt& __x) const
-> decltype(const_iterator{_M_t._M_find_tr(__x)})
{ return const_iterator{_M_t._M_find_tr(__x)}; }
iterator
lower_bound(const key_type& __x)
{ return _M_t.lower_bound(__x); }
const_iterator
lower_bound(const key_type& __x) const
{ return _M_t.lower_bound(__x); }
template<typename _Kt>
auto
lower_bound(const _Kt& __x)
-> decltype(iterator(_M_t._M_lower_bound_tr(__x)))
{ return iterator(_M_t._M_lower_bound_tr(__x)); }
template<typename _Kt>
auto
lower_bound(const _Kt& __x) const
-> decltype(iterator(_M_t._M_lower_bound_tr(__x)))
{ return iterator(_M_t._M_lower_bound_tr(__x)); }
iterator
upper_bound(const key_type& __x)
{ return _M_t.upper_bound(__x); }
const_iterator
upper_bound(const key_type& __x) const
{ return _M_t.upper_bound(__x); }
template<typename _Kt>
auto
upper_bound(const _Kt& __x)
-> decltype(iterator(_M_t._M_upper_bound_tr(__x)))
{ return iterator(_M_t._M_upper_bound_tr(__x)); }
template<typename _Kt>
auto
upper_bound(const _Kt& __x) const
-> decltype(iterator(_M_t._M_upper_bound_tr(__x)))
{ return iterator(_M_t._M_upper_bound_tr(__x)); }
std::pair<iterator, iterator>
equal_range(const key_type& __x)
{ return _M_t.equal_range(__x); }
std::pair<const_iterator, const_iterator>
equal_range(const key_type& __x) const
{ return _M_t.equal_range(__x); }
template<typename _Kt>
auto
equal_range(const _Kt& __x)
-> decltype(pair<iterator, iterator>(_M_t._M_equal_range_tr(__x)))
{ return pair<iterator, iterator>(_M_t._M_equal_range_tr(__x)); }
template<typename _Kt>
auto
equal_range(const _Kt& __x) const
-> decltype(pair<iterator, iterator>(_M_t._M_equal_range_tr(__x)))
{ return pair<iterator, iterator>(_M_t._M_equal_range_tr(__x)); }
template<typename _K1, typename _C1, typename _A1>
friend bool
operator==(const multiset<_K1, _C1, _A1>&,
const multiset<_K1, _C1, _A1>&);
template<typename _K1, typename _C1, typename _A1>
friend bool
operator< (const multiset<_K1, _C1, _A1>&,
const multiset<_K1, _C1, _A1>&);
};
template<typename _InputIterator,
typename _Compare =
less<typename iterator_traits<_InputIterator>::value_type>,
typename _Allocator =
allocator<typename iterator_traits<_InputIterator>::value_type>,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireNotAllocator<_Compare>,
typename = _RequireAllocator<_Allocator>>
multiset(_InputIterator, _InputIterator,
_Compare = _Compare(), _Allocator = _Allocator())
-> multiset<typename iterator_traits<_InputIterator>::value_type,
_Compare, _Allocator>;
template<typename _Key,
typename _Compare = less<_Key>,
typename _Allocator = allocator<_Key>,
typename = _RequireNotAllocator<_Compare>,
typename = _RequireAllocator<_Allocator>>
multiset(initializer_list<_Key>,
_Compare = _Compare(), _Allocator = _Allocator())
-> multiset<_Key, _Compare, _Allocator>;
template<typename _InputIterator, typename _Allocator,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireAllocator<_Allocator>>
multiset(_InputIterator, _InputIterator, _Allocator)
-> multiset<typename iterator_traits<_InputIterator>::value_type,
less<typename iterator_traits<_InputIterator>::value_type>,
_Allocator>;
template<typename _Key, typename _Allocator,
typename = _RequireAllocator<_Allocator>>
multiset(initializer_list<_Key>, _Allocator)
-> multiset<_Key, less<_Key>, _Allocator>;
template<typename _Key, typename _Compare, typename _Alloc>
inline bool
operator==(const multiset<_Key, _Compare, _Alloc>& __x,
const multiset<_Key, _Compare, _Alloc>& __y)
{ return __x._M_t == __y._M_t; }
template<typename _Key, typename _Compare, typename _Alloc>
inline bool
operator<(const multiset<_Key, _Compare, _Alloc>& __x,
const multiset<_Key, _Compare, _Alloc>& __y)
{ return __x._M_t < __y._M_t; }
template<typename _Key, typename _Compare, typename _Alloc>
inline bool
operator!=(const multiset<_Key, _Compare, _Alloc>& __x,
const multiset<_Key, _Compare, _Alloc>& __y)
{ return !(__x == __y); }
template<typename _Key, typename _Compare, typename _Alloc>
inline bool
operator>(const multiset<_Key,_Compare,_Alloc>& __x,
const multiset<_Key,_Compare,_Alloc>& __y)
{ return __y < __x; }
template<typename _Key, typename _Compare, typename _Alloc>
inline bool
operator<=(const multiset<_Key, _Compare, _Alloc>& __x,
const multiset<_Key, _Compare, _Alloc>& __y)
{ return !(__y < __x); }
template<typename _Key, typename _Compare, typename _Alloc>
inline bool
operator>=(const multiset<_Key, _Compare, _Alloc>& __x,
const multiset<_Key, _Compare, _Alloc>& __y)
{ return !(__x < __y); }
template<typename _Key, typename _Compare, typename _Alloc>
inline void
swap(multiset<_Key, _Compare, _Alloc>& __x,
multiset<_Key, _Compare, _Alloc>& __y)
noexcept(noexcept(__x.swap(__y)))
{ __x.swap(__y); }
template<typename _Val, typename _Cmp1, typename _Alloc, typename _Cmp2>
struct
_Rb_tree_merge_helper<std::multiset<_Val, _Cmp1, _Alloc>,
_Cmp2>
{
private:
friend class std::multiset<_Val, _Cmp1, _Alloc>;
static auto&
_S_get_tree(std::set<_Val, _Cmp2, _Alloc>& __set)
{ return __set._M_t; }
static auto&
_S_get_tree(std::multiset<_Val, _Cmp2, _Alloc>& __set)
{ return __set._M_t; }
};
}
namespace std
{
namespace __detail
{
template<typename _Container, typename _UnsafeContainer,
typename _Predicate>
typename _Container::size_type
__erase_nodes_if(_Container& __cont, _UnsafeContainer& __ucont,
_Predicate __pred)
{
typename _Container::size_type __num = 0;
for (auto __iter = __ucont.begin(), __last = __ucont.end();
__iter != __last;)
{
if (__pred(*__iter))
{
__iter = __cont.erase(__iter);
++__num;
}
else
++__iter;
}
return __num;
}
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace pmr
{
template<typename _Key, typename _Cmp = std::less<_Key>>
using set = std::set<_Key, _Cmp, polymorphic_allocator<_Key>>;
template<typename _Key, typename _Cmp = std::less<_Key>>
using multiset = std::multiset<_Key, _Cmp, polymorphic_allocator<_Key>>;
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
extern istream cin;
extern ostream cout;
extern ostream cerr;
extern ostream clog;
extern wistream wcin;
extern wostream wcout;
extern wostream wcerr;
extern wostream wclog;
__extension__ __asm (".globl _ZSt21ios_base_library_initv");
}
extern "C" {
struct stat
{
__dev_t st_dev;
unsigned short int __pad1;
__ino_t st_ino;
__mode_t st_mode;
__nlink_t st_nlink;
__uid_t st_uid;
__gid_t st_gid;
__dev_t st_rdev;
unsigned short int __pad2;
__off_t st_size;
__blksize_t st_blksize;
__blkcnt_t st_blocks;
struct timespec st_atim;
struct timespec st_mtim;
struct timespec st_ctim;
unsigned long int __glibc_reserved4;
unsigned long int __glibc_reserved5;
};
struct stat64
{
__dev_t st_dev;
unsigned int __pad1;
__ino_t __st_ino;
__mode_t st_mode;
__nlink_t st_nlink;
__uid_t st_uid;
__gid_t st_gid;
__dev_t st_rdev;
unsigned int __pad2;
__off64_t st_size;
__blksize_t st_blksize;
__blkcnt64_t st_blocks;
struct timespec st_atim;
struct timespec st_mtim;
struct timespec st_ctim;
__ino64_t st_ino;
};
extern int stat (const char *__restrict __file,
struct stat *__restrict __buf) noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int fstat (int __fd, struct stat *__buf) noexcept (true) __attribute__ ((__nonnull__ (2)));
extern int stat64 (const char *__restrict __file,
struct stat64 *__restrict __buf) noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int fstat64 (int __fd, struct stat64 *__buf) noexcept (true) __attribute__ ((__nonnull__ (2)));
extern int fstatat (int __fd, const char *__restrict __file,
struct stat *__restrict __buf, int __flag)
noexcept (true) __attribute__ ((__nonnull__ (2, 3)));
extern int fstatat64 (int __fd, const char *__restrict __file,
struct stat64 *__restrict __buf, int __flag)
noexcept (true) __attribute__ ((__nonnull__ (2, 3)));
extern int lstat (const char *__restrict __file,
struct stat *__restrict __buf) noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int lstat64 (const char *__restrict __file,
struct stat64 *__restrict __buf)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int chmod (const char *__file, __mode_t __mode)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int lchmod (const char *__file, __mode_t __mode)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int fchmod (int __fd, __mode_t __mode) noexcept (true);
extern int fchmodat (int __fd, const char *__file, __mode_t __mode,
int __flag)
noexcept (true) __attribute__ ((__nonnull__ (2))) ;
extern __mode_t umask (__mode_t __mask) noexcept (true);
extern __mode_t getumask (void) noexcept (true);
extern int mkdir (const char *__path, __mode_t __mode)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int mkdirat (int __fd, const char *__path, __mode_t __mode)
noexcept (true) __attribute__ ((__nonnull__ (2)));
extern int mknod (const char *__path, __mode_t __mode, __dev_t __dev)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int mknodat (int __fd, const char *__path, __mode_t __mode,
__dev_t __dev) noexcept (true) __attribute__ ((__nonnull__ (2)));
extern int mkfifo (const char *__path, __mode_t __mode)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int mkfifoat (int __fd, const char *__path, __mode_t __mode)
noexcept (true) __attribute__ ((__nonnull__ (2)));
extern int utimensat (int __fd, const char *__path,
const struct timespec __times[2],
int __flags)
noexcept (true) __attribute__ ((__nonnull__ (2)));
extern int futimens (int __fd, const struct timespec __times[2]) noexcept (true);
typedef __signed__ char __s8;
typedef unsigned char __u8;
typedef __signed__ short __s16;
typedef unsigned short __u16;
typedef __signed__ int __s32;
typedef unsigned int __u32;
__extension__ typedef __signed__ long long __s64;
__extension__ typedef unsigned long long __u64;
typedef struct {
unsigned long fds_bits[1024 / (8 * sizeof(long))];
} __kernel_fd_set;
typedef void (*__kernel_sighandler_t)(int);
typedef int __kernel_key_t;
typedef int __kernel_mqd_t;
typedef unsigned short __kernel_mode_t;
typedef unsigned short __kernel_ipc_pid_t;
typedef unsigned short __kernel_uid_t;
typedef unsigned short __kernel_gid_t;
typedef unsigned short __kernel_old_dev_t;
typedef long __kernel_long_t;
typedef unsigned long __kernel_ulong_t;
typedef __kernel_ulong_t __kernel_ino_t;
typedef int __kernel_pid_t;
typedef __kernel_long_t __kernel_suseconds_t;
typedef int __kernel_daddr_t;
typedef unsigned int __kernel_uid32_t;
typedef unsigned int __kernel_gid32_t;
typedef __kernel_uid_t __kernel_old_uid_t;
typedef __kernel_gid_t __kernel_old_gid_t;
typedef unsigned int __kernel_size_t;
typedef int __kernel_ssize_t;
typedef int __kernel_ptrdiff_t;
typedef struct {
int val[2];
} __kernel_fsid_t;
typedef __kernel_long_t __kernel_off_t;
typedef long long __kernel_loff_t;
typedef __kernel_long_t __kernel_old_time_t;
typedef __kernel_long_t __kernel_time_t;
typedef long long __kernel_time64_t;
typedef __kernel_long_t __kernel_clock_t;
typedef int __kernel_timer_t;
typedef int __kernel_clockid_t;
typedef char * __kernel_caddr_t;
typedef unsigned short __kernel_uid16_t;
typedef unsigned short __kernel_gid16_t;
typedef __u16 __le16;
typedef __u16 __be16;
typedef __u32 __le32;
typedef __u32 __be32;
typedef __u64 __le64;
typedef __u64 __be64;
typedef __u16 __sum16;
typedef __u32 __wsum;
typedef unsigned __poll_t;
struct statx_timestamp {
__s64 tv_sec;
__u32 tv_nsec;
__s32 __reserved;
};
struct statx {
__u32 stx_mask;
__u32 stx_blksize;
__u64 stx_attributes;
__u32 stx_nlink;
__u32 stx_uid;
__u32 stx_gid;
__u16 stx_mode;
__u16 __spare0[1];
__u64 stx_ino;
__u64 stx_size;
__u64 stx_blocks;
__u64 stx_attributes_mask;
struct statx_timestamp stx_atime;
struct statx_timestamp stx_btime;
struct statx_timestamp stx_ctime;
struct statx_timestamp stx_mtime;
__u32 stx_rdev_major;
__u32 stx_rdev_minor;
__u32 stx_dev_major;
__u32 stx_dev_minor;
__u64 stx_mnt_id;
__u32 stx_dio_mem_align;
__u32 stx_dio_offset_align;
__u64 __spare3[12];
};
extern "C" {
int statx (int __dirfd, const char *__restrict __path, int __flags,
unsigned int __mask, struct statx *__restrict __buf)
noexcept (true) __attribute__ ((__nonnull__ (2, 5)));
}
}
extern "C" {
typedef __socklen_t socklen_t;
extern int access (const char *__name, int __type) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int euidaccess (const char *__name, int __type)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int eaccess (const char *__name, int __type)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int execveat (int __fd, const char *__path, char *const __argv[],
char *const __envp[], int __flags)
noexcept (true) __attribute__ ((__nonnull__ (2, 3)));
extern int faccessat (int __fd, const char *__file, int __type, int __flag)
noexcept (true) __attribute__ ((__nonnull__ (2))) ;
extern __off_t lseek (int __fd, __off_t __offset, int __whence) noexcept (true);
extern __off64_t lseek64 (int __fd, __off64_t __offset, int __whence)
noexcept (true);
extern int close (int __fd);
extern void closefrom (int __lowfd) noexcept (true);
extern ssize_t read (int __fd, void *__buf, size_t __nbytes)
__attribute__ ((__access__ (__write_only__, 2, 3)));
extern ssize_t write (int __fd, const void *__buf, size_t __n)
__attribute__ ((__access__ (__read_only__, 2, 3)));
extern ssize_t pread (int __fd, void *__buf, size_t __nbytes,
__off_t __offset)
__attribute__ ((__access__ (__write_only__, 2, 3)));
extern ssize_t pwrite (int __fd, const void *__buf, size_t __n,
__off_t __offset)
__attribute__ ((__access__ (__read_only__, 2, 3)));
extern ssize_t pread64 (int __fd, void *__buf, size_t __nbytes,
__off64_t __offset)
__attribute__ ((__access__ (__write_only__, 2, 3)));
extern ssize_t pwrite64 (int __fd, const void *__buf, size_t __n,
__off64_t __offset)
__attribute__ ((__access__ (__read_only__, 2, 3)));
extern int pipe (int __pipedes[2]) noexcept (true) ;
extern int pipe2 (int __pipedes[2], int __flags) noexcept (true) ;
extern unsigned int alarm (unsigned int __seconds) noexcept (true);
extern unsigned int sleep (unsigned int __seconds);
extern __useconds_t ualarm (__useconds_t __value, __useconds_t __interval)
noexcept (true);
extern int usleep (__useconds_t __useconds);
extern int pause (void);
extern int chown (const char *__file, __uid_t __owner, __gid_t __group)
noexcept (true) __attribute__ ((__nonnull__ (1))) ;
extern int fchown (int __fd, __uid_t __owner, __gid_t __group) noexcept (true) ;
extern int lchown (const char *__file, __uid_t __owner, __gid_t __group)
noexcept (true) __attribute__ ((__nonnull__ (1))) ;
extern int fchownat (int __fd, const char *__file, __uid_t __owner,
__gid_t __group, int __flag)
noexcept (true) __attribute__ ((__nonnull__ (2))) ;
extern int chdir (const char *__path) noexcept (true) __attribute__ ((__nonnull__ (1))) ;
extern int fchdir (int __fd) noexcept (true) ;
extern char *getcwd (char *__buf, size_t __size) noexcept (true) ;
extern char *get_current_dir_name (void) noexcept (true);
extern char *getwd (char *__buf)
noexcept (true) __attribute__ ((__nonnull__ (1))) __attribute__ ((__deprecated__))
__attribute__ ((__access__ (__write_only__, 1)));
extern int dup (int __fd) noexcept (true) ;
extern int dup2 (int __fd, int __fd2) noexcept (true);
extern int dup3 (int __fd, int __fd2, int __flags) noexcept (true);
extern char **__environ;
extern char **environ;
extern int execve (const char *__path, char *const __argv[],
char *const __envp[]) noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int fexecve (int __fd, char *const __argv[], char *const __envp[])
noexcept (true) __attribute__ ((__nonnull__ (2)));
extern int execv (const char *__path, char *const __argv[])
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int execle (const char *__path, const char *__arg, ...)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int execl (const char *__path, const char *__arg, ...)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int execvp (const char *__file, char *const __argv[])
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int execlp (const char *__file, const char *__arg, ...)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int execvpe (const char *__file, char *const __argv[],
char *const __envp[])
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern int nice (int __inc) noexcept (true) ;
extern void _exit (int __status) __attribute__ ((__noreturn__));
enum
{
_PC_LINK_MAX,
_PC_MAX_CANON,
_PC_MAX_INPUT,
_PC_NAME_MAX,
_PC_PATH_MAX,
_PC_PIPE_BUF,
_PC_CHOWN_RESTRICTED,
_PC_NO_TRUNC,
_PC_VDISABLE,
_PC_SYNC_IO,
_PC_ASYNC_IO,
_PC_PRIO_IO,
_PC_SOCK_MAXBUF,
_PC_FILESIZEBITS,
_PC_REC_INCR_XFER_SIZE,
_PC_REC_MAX_XFER_SIZE,
_PC_REC_MIN_XFER_SIZE,
_PC_REC_XFER_ALIGN,
_PC_ALLOC_SIZE_MIN,
_PC_SYMLINK_MAX,
_PC_2_SYMLINKS
};
enum
{
_SC_ARG_MAX,
_SC_CHILD_MAX,
_SC_CLK_TCK,
_SC_NGROUPS_MAX,
_SC_OPEN_MAX,
_SC_STREAM_MAX,
_SC_TZNAME_MAX,
_SC_JOB_CONTROL,
_SC_SAVED_IDS,
_SC_REALTIME_SIGNALS,
_SC_PRIORITY_SCHEDULING,
_SC_TIMERS,
_SC_ASYNCHRONOUS_IO,
_SC_PRIORITIZED_IO,
_SC_SYNCHRONIZED_IO,
_SC_FSYNC,
_SC_MAPPED_FILES,
_SC_MEMLOCK,
_SC_MEMLOCK_RANGE,
_SC_MEMORY_PROTECTION,
_SC_MESSAGE_PASSING,
_SC_SEMAPHORES,
_SC_SHARED_MEMORY_OBJECTS,
_SC_AIO_LISTIO_MAX,
_SC_AIO_MAX,
_SC_AIO_PRIO_DELTA_MAX,
_SC_DELAYTIMER_MAX,
_SC_MQ_OPEN_MAX,
_SC_MQ_PRIO_MAX,
_SC_VERSION,
_SC_PAGESIZE,
_SC_RTSIG_MAX,
_SC_SEM_NSEMS_MAX,
_SC_SEM_VALUE_MAX,
_SC_SIGQUEUE_MAX,
_SC_TIMER_MAX,
_SC_BC_BASE_MAX,
_SC_BC_DIM_MAX,
_SC_BC_SCALE_MAX,
_SC_BC_STRING_MAX,
_SC_COLL_WEIGHTS_MAX,
_SC_EQUIV_CLASS_MAX,
_SC_EXPR_NEST_MAX,
_SC_LINE_MAX,
_SC_RE_DUP_MAX,
_SC_CHARCLASS_NAME_MAX,
_SC_2_VERSION,
_SC_2_C_BIND,
_SC_2_C_DEV,
_SC_2_FORT_DEV,
_SC_2_FORT_RUN,
_SC_2_SW_DEV,
_SC_2_LOCALEDEF,
_SC_PII,
_SC_PII_XTI,
_SC_PII_SOCKET,
_SC_PII_INTERNET,
_SC_PII_OSI,
_SC_POLL,
_SC_SELECT,
_SC_UIO_MAXIOV,
_SC_IOV_MAX = _SC_UIO_MAXIOV,
_SC_PII_INTERNET_STREAM,
_SC_PII_INTERNET_DGRAM,
_SC_PII_OSI_COTS,
_SC_PII_OSI_CLTS,
_SC_PII_OSI_M,
_SC_T_IOV_MAX,
_SC_THREADS,
_SC_THREAD_SAFE_FUNCTIONS,
_SC_GETGR_R_SIZE_MAX,
_SC_GETPW_R_SIZE_MAX,
_SC_LOGIN_NAME_MAX,
_SC_TTY_NAME_MAX,
_SC_THREAD_DESTRUCTOR_ITERATIONS,
_SC_THREAD_KEYS_MAX,
_SC_THREAD_STACK_MIN,
_SC_THREAD_THREADS_MAX,
_SC_THREAD_ATTR_STACKADDR,
_SC_THREAD_ATTR_STACKSIZE,
_SC_THREAD_PRIORITY_SCHEDULING,
_SC_THREAD_PRIO_INHERIT,
_SC_THREAD_PRIO_PROTECT,
_SC_THREAD_PROCESS_SHARED,
_SC_NPROCESSORS_CONF,
_SC_NPROCESSORS_ONLN,
_SC_PHYS_PAGES,
_SC_AVPHYS_PAGES,
_SC_ATEXIT_MAX,
_SC_PASS_MAX,
_SC_XOPEN_VERSION,
_SC_XOPEN_XCU_VERSION,
_SC_XOPEN_UNIX,
_SC_XOPEN_CRYPT,
_SC_XOPEN_ENH_I18N,
_SC_XOPEN_SHM,
_SC_2_CHAR_TERM,
_SC_2_C_VERSION,
_SC_2_UPE,
_SC_XOPEN_XPG2,
_SC_XOPEN_XPG3,
_SC_XOPEN_XPG4,
_SC_CHAR_BIT,
_SC_CHAR_MAX,
_SC_CHAR_MIN,
_SC_INT_MAX,
_SC_INT_MIN,
_SC_LONG_BIT,
_SC_WORD_BIT,
_SC_MB_LEN_MAX,
_SC_NZERO,
_SC_SSIZE_MAX,
_SC_SCHAR_MAX,
_SC_SCHAR_MIN,
_SC_SHRT_MAX,
_SC_SHRT_MIN,
_SC_UCHAR_MAX,
_SC_UINT_MAX,
_SC_ULONG_MAX,
_SC_USHRT_MAX,
_SC_NL_ARGMAX,
_SC_NL_LANGMAX,
_SC_NL_MSGMAX,
_SC_NL_NMAX,
_SC_NL_SETMAX,
_SC_NL_TEXTMAX,
_SC_XBS5_ILP32_OFF32,
_SC_XBS5_ILP32_OFFBIG,
_SC_XBS5_LP64_OFF64,
_SC_XBS5_LPBIG_OFFBIG,
_SC_XOPEN_LEGACY,
_SC_XOPEN_REALTIME,
_SC_XOPEN_REALTIME_THREADS,
_SC_ADVISORY_INFO,
_SC_BARRIERS,
_SC_BASE,
_SC_C_LANG_SUPPORT,
_SC_C_LANG_SUPPORT_R,
_SC_CLOCK_SELECTION,
_SC_CPUTIME,
_SC_THREAD_CPUTIME,
_SC_DEVICE_IO,
_SC_DEVICE_SPECIFIC,
_SC_DEVICE_SPECIFIC_R,
_SC_FD_MGMT,
_SC_FIFO,
_SC_PIPE,
_SC_FILE_ATTRIBUTES,
_SC_FILE_LOCKING,
_SC_FILE_SYSTEM,
_SC_MONOTONIC_CLOCK,
_SC_MULTI_PROCESS,
_SC_SINGLE_PROCESS,
_SC_NETWORKING,
_SC_READER_WRITER_LOCKS,
_SC_SPIN_LOCKS,
_SC_REGEXP,
_SC_REGEX_VERSION,
_SC_SHELL,
_SC_SIGNALS,
_SC_SPAWN,
_SC_SPORADIC_SERVER,
_SC_THREAD_SPORADIC_SERVER,
_SC_SYSTEM_DATABASE,
_SC_SYSTEM_DATABASE_R,
_SC_TIMEOUTS,
_SC_TYPED_MEMORY_OBJECTS,
_SC_USER_GROUPS,
_SC_USER_GROUPS_R,
_SC_2_PBS,
_SC_2_PBS_ACCOUNTING,
_SC_2_PBS_LOCATE,
_SC_2_PBS_MESSAGE,
_SC_2_PBS_TRACK,
_SC_SYMLOOP_MAX,
_SC_STREAMS,
_SC_2_PBS_CHECKPOINT,
_SC_V6_ILP32_OFF32,
_SC_V6_ILP32_OFFBIG,
_SC_V6_LP64_OFF64,
_SC_V6_LPBIG_OFFBIG,
_SC_HOST_NAME_MAX,
_SC_TRACE,
_SC_TRACE_EVENT_FILTER,
_SC_TRACE_INHERIT,
_SC_TRACE_LOG,
_SC_LEVEL1_ICACHE_SIZE,
_SC_LEVEL1_ICACHE_ASSOC,
_SC_LEVEL1_ICACHE_LINESIZE,
_SC_LEVEL1_DCACHE_SIZE,
_SC_LEVEL1_DCACHE_ASSOC,
_SC_LEVEL1_DCACHE_LINESIZE,
_SC_LEVEL2_CACHE_SIZE,
_SC_LEVEL2_CACHE_ASSOC,
_SC_LEVEL2_CACHE_LINESIZE,
_SC_LEVEL3_CACHE_SIZE,
_SC_LEVEL3_CACHE_ASSOC,
_SC_LEVEL3_CACHE_LINESIZE,
_SC_LEVEL4_CACHE_SIZE,
_SC_LEVEL4_CACHE_ASSOC,
_SC_LEVEL4_CACHE_LINESIZE,
_SC_IPV6 = _SC_LEVEL1_ICACHE_SIZE + 50,
_SC_RAW_SOCKETS,
_SC_V7_ILP32_OFF32,
_SC_V7_ILP32_OFFBIG,
_SC_V7_LP64_OFF64,
_SC_V7_LPBIG_OFFBIG,
_SC_SS_REPL_MAX,
_SC_TRACE_EVENT_NAME_MAX,
_SC_TRACE_NAME_MAX,
_SC_TRACE_SYS_MAX,
_SC_TRACE_USER_EVENT_MAX,
_SC_XOPEN_STREAMS,
_SC_THREAD_ROBUST_PRIO_INHERIT,
_SC_THREAD_ROBUST_PRIO_PROTECT,
_SC_MINSIGSTKSZ,
_SC_SIGSTKSZ
};
enum
{
_CS_PATH,
_CS_V6_WIDTH_RESTRICTED_ENVS,
_CS_GNU_LIBC_VERSION,
_CS_GNU_LIBPTHREAD_VERSION,
_CS_V5_WIDTH_RESTRICTED_ENVS,
_CS_V7_WIDTH_RESTRICTED_ENVS,
_CS_LFS_CFLAGS = 1000,
_CS_LFS_LDFLAGS,
_CS_LFS_LIBS,
_CS_LFS_LINTFLAGS,
_CS_LFS64_CFLAGS,
_CS_LFS64_LDFLAGS,
_CS_LFS64_LIBS,
_CS_LFS64_LINTFLAGS,
_CS_XBS5_ILP32_OFF32_CFLAGS = 1100,
_CS_XBS5_ILP32_OFF32_LDFLAGS,
_CS_XBS5_ILP32_OFF32_LIBS,
_CS_XBS5_ILP32_OFF32_LINTFLAGS,
_CS_XBS5_ILP32_OFFBIG_CFLAGS,
_CS_XBS5_ILP32_OFFBIG_LDFLAGS,
_CS_XBS5_ILP32_OFFBIG_LIBS,
_CS_XBS5_ILP32_OFFBIG_LINTFLAGS,
_CS_XBS5_LP64_OFF64_CFLAGS,
_CS_XBS5_LP64_OFF64_LDFLAGS,
_CS_XBS5_LP64_OFF64_LIBS,
_CS_XBS5_LP64_OFF64_LINTFLAGS,
_CS_XBS5_LPBIG_OFFBIG_CFLAGS,
_CS_XBS5_LPBIG_OFFBIG_LDFLAGS,
_CS_XBS5_LPBIG_OFFBIG_LIBS,
_CS_XBS5_LPBIG_OFFBIG_LINTFLAGS,
_CS_POSIX_V6_ILP32_OFF32_CFLAGS,
_CS_POSIX_V6_ILP32_OFF32_LDFLAGS,
_CS_POSIX_V6_ILP32_OFF32_LIBS,
_CS_POSIX_V6_ILP32_OFF32_LINTFLAGS,
_CS_POSIX_V6_ILP32_OFFBIG_CFLAGS,
_CS_POSIX_V6_ILP32_OFFBIG_LDFLAGS,
_CS_POSIX_V6_ILP32_OFFBIG_LIBS,
_CS_POSIX_V6_ILP32_OFFBIG_LINTFLAGS,
_CS_POSIX_V6_LP64_OFF64_CFLAGS,
_CS_POSIX_V6_LP64_OFF64_LDFLAGS,
_CS_POSIX_V6_LP64_OFF64_LIBS,
_CS_POSIX_V6_LP64_OFF64_LINTFLAGS,
_CS_POSIX_V6_LPBIG_OFFBIG_CFLAGS,
_CS_POSIX_V6_LPBIG_OFFBIG_LDFLAGS,
_CS_POSIX_V6_LPBIG_OFFBIG_LIBS,
_CS_POSIX_V6_LPBIG_OFFBIG_LINTFLAGS,
_CS_POSIX_V7_ILP32_OFF32_CFLAGS,
_CS_POSIX_V7_ILP32_OFF32_LDFLAGS,
_CS_POSIX_V7_ILP32_OFF32_LIBS,
_CS_POSIX_V7_ILP32_OFF32_LINTFLAGS,
_CS_POSIX_V7_ILP32_OFFBIG_CFLAGS,
_CS_POSIX_V7_ILP32_OFFBIG_LDFLAGS,
_CS_POSIX_V7_ILP32_OFFBIG_LIBS,
_CS_POSIX_V7_ILP32_OFFBIG_LINTFLAGS,
_CS_POSIX_V7_LP64_OFF64_CFLAGS,
_CS_POSIX_V7_LP64_OFF64_LDFLAGS,
_CS_POSIX_V7_LP64_OFF64_LIBS,
_CS_POSIX_V7_LP64_OFF64_LINTFLAGS,
_CS_POSIX_V7_LPBIG_OFFBIG_CFLAGS,
_CS_POSIX_V7_LPBIG_OFFBIG_LDFLAGS,
_CS_POSIX_V7_LPBIG_OFFBIG_LIBS,
_CS_POSIX_V7_LPBIG_OFFBIG_LINTFLAGS,
_CS_V6_ENV,
_CS_V7_ENV
};
extern long int pathconf (const char *__path, int __name)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern long int fpathconf (int __fd, int __name) noexcept (true);
extern long int sysconf (int __name) noexcept (true);
extern size_t confstr (int __name, char *__buf, size_t __len) noexcept (true)
__attribute__ ((__access__ (__write_only__, 2, 3)));
extern __pid_t getpid (void) noexcept (true);
extern __pid_t getppid (void) noexcept (true);
extern __pid_t getpgrp (void) noexcept (true);
extern __pid_t __getpgid (__pid_t __pid) noexcept (true);
extern __pid_t getpgid (__pid_t __pid) noexcept (true);
extern int setpgid (__pid_t __pid, __pid_t __pgid) noexcept (true);
extern int setpgrp (void) noexcept (true);
extern __pid_t setsid (void) noexcept (true);
extern __pid_t getsid (__pid_t __pid) noexcept (true);
extern __uid_t getuid (void) noexcept (true);
extern __uid_t geteuid (void) noexcept (true);
extern __gid_t getgid (void) noexcept (true);
extern __gid_t getegid (void) noexcept (true);
extern int getgroups (int __size, __gid_t __list[]) noexcept (true)
__attribute__ ((__access__ (__write_only__, 2, 1)));
extern int group_member (__gid_t __gid) noexcept (true);
extern int setuid (__uid_t __uid) noexcept (true) ;
extern int setreuid (__uid_t __ruid, __uid_t __euid) noexcept (true) ;
extern int seteuid (__uid_t __uid) noexcept (true) ;
extern int setgid (__gid_t __gid) noexcept (true) ;
extern int setregid (__gid_t __rgid, __gid_t __egid) noexcept (true) ;
extern int setegid (__gid_t __gid) noexcept (true) ;
extern int getresuid (__uid_t *__ruid, __uid_t *__euid, __uid_t *__suid)
noexcept (true);
extern int getresgid (__gid_t *__rgid, __gid_t *__egid, __gid_t *__sgid)
noexcept (true);
extern int setresuid (__uid_t __ruid, __uid_t __euid, __uid_t __suid)
noexcept (true) ;
extern int setresgid (__gid_t __rgid, __gid_t __egid, __gid_t __sgid)
noexcept (true) ;
extern __pid_t fork (void) noexcept (true);
extern __pid_t vfork (void) noexcept (true);
extern __pid_t _Fork (void) noexcept (true);
extern char *ttyname (int __fd) noexcept (true);
extern int ttyname_r (int __fd, char *__buf, size_t __buflen)
noexcept (true) __attribute__ ((__nonnull__ (2)))
__attribute__ ((__access__ (__write_only__, 2, 3)));
extern int isatty (int __fd) noexcept (true);
extern int ttyslot (void) noexcept (true);
extern int link (const char *__from, const char *__to)
noexcept (true) __attribute__ ((__nonnull__ (1, 2))) ;
extern int linkat (int __fromfd, const char *__from, int __tofd,
const char *__to, int __flags)
noexcept (true) __attribute__ ((__nonnull__ (2, 4))) ;
extern int symlink (const char *__from, const char *__to)
noexcept (true) __attribute__ ((__nonnull__ (1, 2))) ;
extern ssize_t readlink (const char *__restrict __path,
char *__restrict __buf, size_t __len)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)))
__attribute__ ((__access__ (__write_only__, 2, 3)));
extern int symlinkat (const char *__from, int __tofd,
const char *__to) noexcept (true) __attribute__ ((__nonnull__ (1, 3))) ;
extern ssize_t readlinkat (int __fd, const char *__restrict __path,
char *__restrict __buf, size_t __len)
noexcept (true) __attribute__ ((__nonnull__ (2, 3)))
__attribute__ ((__access__ (__write_only__, 3, 4)));
extern int unlink (const char *__name) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int unlinkat (int __fd, const char *__name, int __flag)
noexcept (true) __attribute__ ((__nonnull__ (2)));
extern int rmdir (const char *__path) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern __pid_t tcgetpgrp (int __fd) noexcept (true);
extern int tcsetpgrp (int __fd, __pid_t __pgrp_id) noexcept (true);
extern char *getlogin (void);
extern int getlogin_r (char *__name, size_t __name_len) __attribute__ ((__nonnull__ (1)))
__attribute__ ((__access__ (__write_only__, 1, 2)));
extern int setlogin (const char *__name) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern "C" {
extern char *optarg;
extern int optind;
extern int opterr;
extern int optopt;
extern int getopt (int ___argc, char *const *___argv, const char *__shortopts)
noexcept (true) __attribute__ ((__nonnull__ (2, 3)));
}
extern "C" {
}
extern int gethostname (char *__name, size_t __len) noexcept (true) __attribute__ ((__nonnull__ (1)))
__attribute__ ((__access__ (__write_only__, 1, 2)));
extern int sethostname (const char *__name, size_t __len)
noexcept (true) __attribute__ ((__nonnull__ (1))) __attribute__ ((__access__ (__read_only__, 1, 2)));
extern int sethostid (long int __id) noexcept (true) ;
extern int getdomainname (char *__name, size_t __len)
noexcept (true) __attribute__ ((__nonnull__ (1)))
__attribute__ ((__access__ (__write_only__, 1, 2)));
extern int setdomainname (const char *__name, size_t __len)
noexcept (true) __attribute__ ((__nonnull__ (1))) __attribute__ ((__access__ (__read_only__, 1, 2)));
extern int vhangup (void) noexcept (true);
extern int revoke (const char *__file) noexcept (true) __attribute__ ((__nonnull__ (1))) ;
extern int profil (unsigned short int *__sample_buffer, size_t __size,
size_t __offset, unsigned int __scale)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int acct (const char *__name) noexcept (true);
extern char *getusershell (void) noexcept (true);
extern void endusershell (void) noexcept (true);
extern void setusershell (void) noexcept (true);
extern int daemon (int __nochdir, int __noclose) noexcept (true) ;
extern int chroot (const char *__path) noexcept (true) __attribute__ ((__nonnull__ (1))) ;
extern char *getpass (const char *__prompt) __attribute__ ((__nonnull__ (1)));
extern int fsync (int __fd);
extern int syncfs (int __fd) noexcept (true);
extern long int gethostid (void);
extern void sync (void) noexcept (true);
extern int getpagesize (void) noexcept (true) __attribute__ ((__const__));
extern int getdtablesize (void) noexcept (true);
extern int truncate (const char *__file, __off_t __length)
noexcept (true) __attribute__ ((__nonnull__ (1))) ;
extern int truncate64 (const char *__file, __off64_t __length)
noexcept (true) __attribute__ ((__nonnull__ (1))) ;
extern int ftruncate (int __fd, __off_t __length) noexcept (true) ;
extern int ftruncate64 (int __fd, __off64_t __length) noexcept (true) ;
extern int brk (void *__addr) noexcept (true) ;
extern void *sbrk (intptr_t __delta) noexcept (true);
extern long int syscall (long int __sysno, ...) noexcept (true);
extern int lockf (int __fd, int __cmd, __off_t __len) ;
extern int lockf64 (int __fd, int __cmd, __off64_t __len) ;
ssize_t copy_file_range (int __infd, __off64_t *__pinoff,
int __outfd, __off64_t *__poutoff,
size_t __length, unsigned int __flags);
extern int fdatasync (int __fildes);
extern char *crypt (const char *__key, const char *__salt)
noexcept (true) __attribute__ ((__nonnull__ (1, 2)));
extern void swab (const void *__restrict __from, void *__restrict __to,
ssize_t __n) noexcept (true) __attribute__ ((__nonnull__ (1, 2)))
__attribute__ ((__access__ (__read_only__, 1, 3)))
__attribute__ ((__access__ (__write_only__, 2, 3)));
int getentropy (void *__buffer, size_t __length)
__attribute__ ((__access__ (__write_only__, 1, 2)));
extern int close_range (unsigned int __fd, unsigned int __max_fd,
int __flags) noexcept (true);
extern __pid_t gettid (void) noexcept (true);
}
extern "C" {
typedef unsigned int __re_size_t;
typedef unsigned long int __re_long_size_t;
typedef long int s_reg_t;
typedef unsigned long int active_reg_t;
typedef unsigned long int reg_syntax_t;
extern reg_syntax_t re_syntax_options;
typedef enum
{
_REG_ENOSYS = -1,
_REG_NOERROR = 0,
_REG_NOMATCH,
_REG_BADPAT,
_REG_ECOLLATE,
_REG_ECTYPE,
_REG_EESCAPE,
_REG_ESUBREG,
_REG_EBRACK,
_REG_EPAREN,
_REG_EBRACE,
_REG_BADBR,
_REG_ERANGE,
_REG_ESPACE,
_REG_BADRPT,
_REG_EEND,
_REG_ESIZE,
_REG_ERPAREN
} reg_errcode_t;
struct re_pattern_buffer
{
struct re_dfa_t *buffer;
__re_long_size_t allocated;
__re_long_size_t used;
reg_syntax_t syntax;
char *fastmap;
unsigned char * translate;
size_t re_nsub;
unsigned can_be_null : 1;
unsigned regs_allocated : 2;
unsigned fastmap_accurate : 1;
unsigned no_sub : 1;
unsigned not_bol : 1;
unsigned not_eol : 1;
unsigned newline_anchor : 1;
};
typedef struct re_pattern_buffer regex_t;
typedef int regoff_t;
struct re_registers
{
__re_size_t num_regs;
regoff_t *start;
regoff_t *end;
};
typedef struct
{
regoff_t rm_so;
regoff_t rm_eo;
} regmatch_t;
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wvla"
extern reg_syntax_t re_set_syntax (reg_syntax_t __syntax);
extern const char *re_compile_pattern (const char *__pattern, size_t __length,
struct re_pattern_buffer *__buffer)
__attribute__ ((__access__ (__read_only__, 1, 2)));
extern int re_compile_fastmap (struct re_pattern_buffer *__buffer);
extern regoff_t re_search (struct re_pattern_buffer *__buffer,
const char *__String, regoff_t __length,
regoff_t __start, regoff_t __range,
struct re_registers *__regs)
__attribute__ ((__access__ (__read_only__, 2, 3)));
extern regoff_t re_search_2 (struct re_pattern_buffer *__buffer,
const char *__string1, regoff_t __length1,
const char *__string2, regoff_t __length2,
regoff_t __start, regoff_t __range,
struct re_registers *__regs,
regoff_t __stop)
__attribute__ ((__access__ (__read_only__, 2, 3)))
__attribute__ ((__access__ (__read_only__, 4, 5)));
extern regoff_t re_match (struct re_pattern_buffer *__buffer,
const char *__String, regoff_t __length,
regoff_t __start, struct re_registers *__regs)
__attribute__ ((__access__ (__read_only__, 2, 3)));
extern regoff_t re_match_2 (struct re_pattern_buffer *__buffer,
const char *__string1, regoff_t __length1,
const char *__string2, regoff_t __length2,
regoff_t __start, struct re_registers *__regs,
regoff_t __stop)
__attribute__ ((__access__ (__read_only__, 2, 3)))
__attribute__ ((__access__ (__read_only__, 4, 5)));
extern void re_set_registers (struct re_pattern_buffer *__buffer,
struct re_registers *__regs,
__re_size_t __num_regs,
regoff_t *__starts, regoff_t *__ends);
extern int regcomp (regex_t *__restrict __preg,
const char *__restrict __pattern,
int __cflags);
extern int regexec (const regex_t *__restrict __preg,
const char *__restrict __String, size_t __nmatch,
regmatch_t __pmatch[
],
int __eflags);
extern size_t regerror (int __errcode, const regex_t *__restrict __preg,
char *__restrict __errbuf, size_t __errbuf_size)
__attribute__ ((__access__ (__write_only__, 3, 4)));
extern void regfree (regex_t *__preg);
#pragma GCC diagnostic pop
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<intmax_t _Pn>
struct __static_sign
: integral_constant<intmax_t, (_Pn < 0) ? -1 : 1>
{ };
template<intmax_t _Pn>
struct __static_abs
: integral_constant<intmax_t, _Pn * __static_sign<_Pn>::value>
{ };
template<intmax_t _Pn, intmax_t _Qn>
struct __static_gcd
: __static_gcd<_Qn, (_Pn % _Qn)>
{ };
template<intmax_t _Pn>
struct __static_gcd<_Pn, 0>
: integral_constant<intmax_t, __static_abs<_Pn>::value>
{ };
template<intmax_t _Qn>
struct __static_gcd<0, _Qn>
: integral_constant<intmax_t, __static_abs<_Qn>::value>
{ };
template<intmax_t _Pn, intmax_t _Qn>
struct __safe_multiply
{
private:
static const uintmax_t __c = uintmax_t(1) << (sizeof(intmax_t) * 4);
static const uintmax_t __a0 = __static_abs<_Pn>::value % __c;
static const uintmax_t __a1 = __static_abs<_Pn>::value / __c;
static const uintmax_t __b0 = __static_abs<_Qn>::value % __c;
static const uintmax_t __b1 = __static_abs<_Qn>::value / __c;
static_assert(__a1 == 0 || __b1 == 0,
"overflow in multiplication");
static_assert(__a0 * __b1 + __b0 * __a1 < (__c >> 1),
"overflow in multiplication");
static_assert(__b0 * __a0 <= 0x7fffffffffffffffLL,
"overflow in multiplication");
static_assert((__a0 * __b1 + __b0 * __a1) * __c
<= 0x7fffffffffffffffLL - __b0 * __a0,
"overflow in multiplication");
public:
static const intmax_t value = _Pn * _Qn;
};
template<uintmax_t __hi1, uintmax_t __lo1, uintmax_t __hi2, uintmax_t __lo2>
struct __big_less
: integral_constant<bool, (__hi1 < __hi2
|| (__hi1 == __hi2 && __lo1 < __lo2))>
{ };
template<uintmax_t __hi1, uintmax_t __lo1, uintmax_t __hi2, uintmax_t __lo2>
struct __big_add
{
static constexpr uintmax_t __lo = __lo1 + __lo2;
static constexpr uintmax_t __hi = (__hi1 + __hi2 +
(__lo1 + __lo2 < __lo1));
};
template<uintmax_t __hi1, uintmax_t __lo1, uintmax_t __hi2, uintmax_t __lo2>
struct __big_sub
{
static_assert(!__big_less<__hi1, __lo1, __hi2, __lo2>::value,
"Internal library error");
static constexpr uintmax_t __lo = __lo1 - __lo2;
static constexpr uintmax_t __hi = (__hi1 - __hi2 -
(__lo1 < __lo2));
};
template<uintmax_t __x, uintmax_t __y>
struct __big_mul
{
private:
static constexpr uintmax_t __c = uintmax_t(1) << (sizeof(intmax_t) * 4);
static constexpr uintmax_t __x0 = __x % __c;
static constexpr uintmax_t __x1 = __x / __c;
static constexpr uintmax_t __y0 = __y % __c;
static constexpr uintmax_t __y1 = __y / __c;
static constexpr uintmax_t __x0y0 = __x0 * __y0;
static constexpr uintmax_t __x0y1 = __x0 * __y1;
static constexpr uintmax_t __x1y0 = __x1 * __y0;
static constexpr uintmax_t __x1y1 = __x1 * __y1;
static constexpr uintmax_t __mix = __x0y1 + __x1y0;
static constexpr uintmax_t __mix_lo = __mix * __c;
static constexpr uintmax_t __mix_hi
= __mix / __c + ((__mix < __x0y1) ? __c : 0);
typedef __big_add<__mix_hi, __mix_lo, __x1y1, __x0y0> _Res;
public:
static constexpr uintmax_t __hi = _Res::__hi;
static constexpr uintmax_t __lo = _Res::__lo;
};
template<uintmax_t __n1, uintmax_t __n0, uintmax_t __d>
struct __big_div_impl
{
private:
static_assert(__d >= (uintmax_t(1) << (sizeof(intmax_t) * 8 - 1)),
"Internal library error");
static_assert(__n1 < __d, "Internal library error");
static constexpr uintmax_t __c = uintmax_t(1) << (sizeof(intmax_t) * 4);
static constexpr uintmax_t __d1 = __d / __c;
static constexpr uintmax_t __d0 = __d % __c;
static constexpr uintmax_t __q1x = __n1 / __d1;
static constexpr uintmax_t __r1x = __n1 % __d1;
static constexpr uintmax_t __m = __q1x * __d0;
static constexpr uintmax_t __r1y = __r1x * __c + __n0 / __c;
static constexpr uintmax_t __r1z = __r1y + __d;
static constexpr uintmax_t __r1
= ((__r1y < __m) ? ((__r1z >= __d) && (__r1z < __m))
? (__r1z + __d) : __r1z : __r1y) - __m;
static constexpr uintmax_t __q1
= __q1x - ((__r1y < __m)
? ((__r1z >= __d) && (__r1z < __m)) ? 2 : 1 : 0);
static constexpr uintmax_t __q0x = __r1 / __d1;
static constexpr uintmax_t __r0x = __r1 % __d1;
static constexpr uintmax_t __n = __q0x * __d0;
static constexpr uintmax_t __r0y = __r0x * __c + __n0 % __c;
static constexpr uintmax_t __r0z = __r0y + __d;
static constexpr uintmax_t __r0
= ((__r0y < __n) ? ((__r0z >= __d) && (__r0z < __n))
? (__r0z + __d) : __r0z : __r0y) - __n;
static constexpr uintmax_t __q0
= __q0x - ((__r0y < __n) ? ((__r0z >= __d)
&& (__r0z < __n)) ? 2 : 1 : 0);
public:
static constexpr uintmax_t __quot = __q1 * __c + __q0;
static constexpr uintmax_t __rem = __r0;
private:
typedef __big_mul<__quot, __d> _Prod;
typedef __big_add<_Prod::__hi, _Prod::__lo, 0, __rem> _Sum;
static_assert(_Sum::__hi == __n1 && _Sum::__lo == __n0,
"Internal library error");
};
template<uintmax_t __n1, uintmax_t __n0, uintmax_t __d>
struct __big_div
{
private:
static_assert(__d != 0, "Internal library error");
static_assert(sizeof (uintmax_t) == sizeof (unsigned long long),
"This library calls __builtin_clzll on uintmax_t, which "
"is unsafe on your platform. Please complain to "
"http://gcc.gnu.org/bugzilla/");
static constexpr int __shift = __builtin_clzll(__d);
static constexpr int __coshift_ = sizeof(uintmax_t) * 8 - __shift;
static constexpr int __coshift = (__shift != 0) ? __coshift_ : 0;
static constexpr uintmax_t __c1 = uintmax_t(1) << __shift;
static constexpr uintmax_t __c2 = uintmax_t(1) << __coshift;
static constexpr uintmax_t __new_d = __d * __c1;
static constexpr uintmax_t __new_n0 = __n0 * __c1;
static constexpr uintmax_t __n1_shifted = (__n1 % __d) * __c1;
static constexpr uintmax_t __n0_top = (__shift != 0) ? (__n0 / __c2) : 0;
static constexpr uintmax_t __new_n1 = __n1_shifted + __n0_top;
typedef __big_div_impl<__new_n1, __new_n0, __new_d> _Res;
public:
static constexpr uintmax_t __quot_hi = __n1 / __d;
static constexpr uintmax_t __quot_lo = _Res::__quot;
static constexpr uintmax_t __rem = _Res::__rem / __c1;
private:
typedef __big_mul<__quot_lo, __d> _P0;
typedef __big_mul<__quot_hi, __d> _P1;
typedef __big_add<_P0::__hi, _P0::__lo, _P1::__lo, __rem> _Sum;
static_assert(_P1::__hi == 0, "Internal library error");
static_assert(_Sum::__hi >= _P0::__hi, "Internal library error");
static_assert(_Sum::__hi == __n1 && _Sum::__lo == __n0,
"Internal library error");
static_assert(__rem < __d, "Internal library error");
};
template<intmax_t _Num, intmax_t _Den = 1>
struct ratio
{
static_assert(_Den != 0, "denominator cannot be zero");
static_assert(_Num >= -0x7fffffffffffffffLL && _Den >= -0x7fffffffffffffffLL,
"out of range");
static constexpr intmax_t num =
_Num * __static_sign<_Den>::value / __static_gcd<_Num, _Den>::value;
static constexpr intmax_t den =
__static_abs<_Den>::value / __static_gcd<_Num, _Den>::value;
typedef ratio<num, den> type;
};
template<typename _Tp>
struct __is_ratio
: std::false_type
{ };
template<intmax_t _Num, intmax_t _Den>
struct __is_ratio<ratio<_Num, _Den>>
: std::true_type
{ };
template<typename _Tp>
constexpr bool __is_ratio_v = false;
template<intmax_t _Num, intmax_t _Den>
constexpr bool __is_ratio_v<ratio<_Num, _Den>> = true;
template<typename _R1, typename _R2>
constexpr bool
__are_both_ratios() noexcept
{
if constexpr (__is_ratio_v<_R1>)
if constexpr (__is_ratio_v<_R2>)
return true;
return false;
}
template<typename _R1, typename _R2>
struct __ratio_multiply
{
static_assert(std::__are_both_ratios<_R1, _R2>(),
"both template arguments must be a std::ratio");
private:
static const intmax_t __gcd1 =
__static_gcd<_R1::num, _R2::den>::value;
static const intmax_t __gcd2 =
__static_gcd<_R2::num, _R1::den>::value;
public:
typedef ratio<
__safe_multiply<(_R1::num / __gcd1),
(_R2::num / __gcd2)>::value,
__safe_multiply<(_R1::den / __gcd2),
(_R2::den / __gcd1)>::value> type;
static constexpr intmax_t num = type::num;
static constexpr intmax_t den = type::den;
};
template<typename _R1, typename _R2>
using ratio_multiply = typename __ratio_multiply<_R1, _R2>::type;
template<typename _R1, typename _R2>
struct __ratio_divide
{
static_assert(_R2::num != 0, "division by 0");
typedef typename __ratio_multiply<
_R1,
ratio<_R2::den, _R2::num>>::type type;
static constexpr intmax_t num = type::num;
static constexpr intmax_t den = type::den;
};
template<typename _R1, typename _R2>
using ratio_divide = typename __ratio_divide<_R1, _R2>::type;
template<typename _R1, typename _R2>
struct ratio_equal
: integral_constant<bool, _R1::num == _R2::num && _R1::den == _R2::den>
{
static_assert(std::__are_both_ratios<_R1, _R2>(),
"both template arguments must be a std::ratio");
};
template<typename _R1, typename _R2>
struct ratio_not_equal
: integral_constant<bool, !ratio_equal<_R1, _R2>::value>
{ };
template<typename _R1, typename _R2,
typename _Left = __big_mul<_R1::num,_R2::den>,
typename _Right = __big_mul<_R2::num,_R1::den> >
struct __ratio_less_impl_1
: integral_constant<bool, __big_less<_Left::__hi, _Left::__lo,
_Right::__hi, _Right::__lo>::value>
{ };
template<typename _R1, typename _R2,
bool = (_R1::num == 0 || _R2::num == 0
|| (__static_sign<_R1::num>::value
!= __static_sign<_R2::num>::value)),
bool = (__static_sign<_R1::num>::value == -1
&& __static_sign<_R2::num>::value == -1)>
struct __ratio_less_impl
: __ratio_less_impl_1<_R1, _R2>::type
{ };
template<typename _R1, typename _R2>
struct __ratio_less_impl<_R1, _R2, true, false>
: integral_constant<bool, _R1::num < _R2::num>
{ };
template<typename _R1, typename _R2>
struct __ratio_less_impl<_R1, _R2, false, true>
: __ratio_less_impl_1<ratio<-_R2::num, _R2::den>,
ratio<-_R1::num, _R1::den> >::type
{ };
template<typename _R1, typename _R2>
struct ratio_less
: __ratio_less_impl<_R1, _R2>::type
{
static_assert(std::__are_both_ratios<_R1, _R2>(),
"both template arguments must be a std::ratio");
};
template<typename _R1, typename _R2>
struct ratio_less_equal
: integral_constant<bool, !ratio_less<_R2, _R1>::value>
{ };
template<typename _R1, typename _R2>
struct ratio_greater
: integral_constant<bool, ratio_less<_R2, _R1>::value>
{ };
template<typename _R1, typename _R2>
struct ratio_greater_equal
: integral_constant<bool, !ratio_less<_R1, _R2>::value>
{ };
template <typename _R1, typename _R2>
inline constexpr bool ratio_equal_v = ratio_equal<_R1, _R2>::value;
template <typename _R1, typename _R2>
inline constexpr bool ratio_not_equal_v = ratio_not_equal<_R1, _R2>::value;
template <typename _R1, typename _R2>
inline constexpr bool ratio_less_v = ratio_less<_R1, _R2>::value;
template <typename _R1, typename _R2>
inline constexpr bool ratio_less_equal_v
= ratio_less_equal<_R1, _R2>::value;
template <typename _R1, typename _R2>
inline constexpr bool ratio_greater_v = ratio_greater<_R1, _R2>::value;
template <typename _R1, typename _R2>
inline constexpr bool ratio_greater_equal_v
= ratio_greater_equal<_R1, _R2>::value;
template<typename _R1, typename _R2,
bool = (_R1::num >= 0),
bool = (_R2::num >= 0),
bool = ratio_less<ratio<__static_abs<_R1::num>::value, _R1::den>,
ratio<__static_abs<_R2::num>::value, _R2::den> >::value>
struct __ratio_add_impl
{
private:
typedef typename __ratio_add_impl<
ratio<-_R1::num, _R1::den>,
ratio<-_R2::num, _R2::den> >::type __t;
public:
typedef ratio<-__t::num, __t::den> type;
};
template<typename _R1, typename _R2, bool __b>
struct __ratio_add_impl<_R1, _R2, true, true, __b>
{
private:
static constexpr uintmax_t __g = __static_gcd<_R1::den, _R2::den>::value;
static constexpr uintmax_t __d2 = _R2::den / __g;
typedef __big_mul<_R1::den, __d2> __d;
typedef __big_mul<_R1::num, _R2::den / __g> __x;
typedef __big_mul<_R2::num, _R1::den / __g> __y;
typedef __big_add<__x::__hi, __x::__lo, __y::__hi, __y::__lo> __n;
static_assert(__n::__hi >= __x::__hi, "Internal library error");
typedef __big_div<__n::__hi, __n::__lo, __g> __ng;
static constexpr uintmax_t __g2 = __static_gcd<__ng::__rem, __g>::value;
typedef __big_div<__n::__hi, __n::__lo, __g2> __n_final;
static_assert(__n_final::__rem == 0, "Internal library error");
static_assert(__n_final::__quot_hi == 0 &&
__n_final::__quot_lo <= 0x7fffffffffffffffLL, "overflow in addition");
typedef __big_mul<_R1::den / __g2, __d2> __d_final;
static_assert(__d_final::__hi == 0 &&
__d_final::__lo <= 0x7fffffffffffffffLL, "overflow in addition");
public:
typedef ratio<__n_final::__quot_lo, __d_final::__lo> type;
};
template<typename _R1, typename _R2>
struct __ratio_add_impl<_R1, _R2, false, true, true>
: __ratio_add_impl<_R2, _R1>
{ };
template<typename _R1, typename _R2>
struct __ratio_add_impl<_R1, _R2, true, false, false>
{
private:
static constexpr uintmax_t __g = __static_gcd<_R1::den, _R2::den>::value;
static constexpr uintmax_t __d2 = _R2::den / __g;
typedef __big_mul<_R1::den, __d2> __d;
typedef __big_mul<_R1::num, _R2::den / __g> __x;
typedef __big_mul<-_R2::num, _R1::den / __g> __y;
typedef __big_sub<__x::__hi, __x::__lo, __y::__hi, __y::__lo> __n;
typedef __big_div<__n::__hi, __n::__lo, __g> __ng;
static constexpr uintmax_t __g2 = __static_gcd<__ng::__rem, __g>::value;
typedef __big_div<__n::__hi, __n::__lo, __g2> __n_final;
static_assert(__n_final::__rem == 0, "Internal library error");
static_assert(__n_final::__quot_hi == 0 &&
__n_final::__quot_lo <= 0x7fffffffffffffffLL, "overflow in addition");
typedef __big_mul<_R1::den / __g2, __d2> __d_final;
static_assert(__d_final::__hi == 0 &&
__d_final::__lo <= 0x7fffffffffffffffLL, "overflow in addition");
public:
typedef ratio<__n_final::__quot_lo, __d_final::__lo> type;
};
template<typename _R1, typename _R2>
struct __ratio_add
{
static_assert(std::__are_both_ratios<_R1, _R2>(),
"both template arguments must be a std::ratio");
typedef typename __ratio_add_impl<_R1, _R2>::type type;
static constexpr intmax_t num = type::num;
static constexpr intmax_t den = type::den;
};
template<typename _R1, typename _R2>
using ratio_add = typename __ratio_add<_R1, _R2>::type;
template<typename _R1, typename _R2>
struct __ratio_subtract
{
typedef typename __ratio_add<
_R1,
ratio<-_R2::num, _R2::den>>::type type;
static constexpr intmax_t num = type::num;
static constexpr intmax_t den = type::den;
};
template<typename _R1, typename _R2>
using ratio_subtract = typename __ratio_subtract<_R1, _R2>::type;
typedef ratio<1, 1000000000000000000> atto;
typedef ratio<1, 1000000000000000> femto;
typedef ratio<1, 1000000000000> pico;
typedef ratio<1, 1000000000> nano;
typedef ratio<1, 1000000> micro;
typedef ratio<1, 1000> milli;
typedef ratio<1, 100> centi;
typedef ratio<1, 10> deci;
typedef ratio< 10, 1> deca;
typedef ratio< 100, 1> hecto;
typedef ratio< 1000, 1> kilo;
typedef ratio< 1000000, 1> mega;
typedef ratio< 1000000000, 1> giga;
typedef ratio< 1000000000000, 1> tera;
typedef ratio< 1000000000000000, 1> peta;
typedef ratio< 1000000000000000000, 1> exa;
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __parse_int
{
template<unsigned _Base, char _Dig>
struct _Digit;
template<unsigned _Base>
struct _Digit<_Base, '0'> : integral_constant<unsigned, 0>
{
using __valid = true_type;
};
template<unsigned _Base>
struct _Digit<_Base, '1'> : integral_constant<unsigned, 1>
{
using __valid = true_type;
};
template<unsigned _Base, unsigned _Val>
struct _Digit_impl : integral_constant<unsigned, _Val>
{
static_assert(_Base > _Val, "invalid digit");
using __valid = true_type;
};
template<unsigned _Base>
struct _Digit<_Base, '2'> : _Digit_impl<_Base, 2>
{ };
template<unsigned _Base>
struct _Digit<_Base, '3'> : _Digit_impl<_Base, 3>
{ };
template<unsigned _Base>
struct _Digit<_Base, '4'> : _Digit_impl<_Base, 4>
{ };
template<unsigned _Base>
struct _Digit<_Base, '5'> : _Digit_impl<_Base, 5>
{ };
template<unsigned _Base>
struct _Digit<_Base, '6'> : _Digit_impl<_Base, 6>
{ };
template<unsigned _Base>
struct _Digit<_Base, '7'> : _Digit_impl<_Base, 7>
{ };
template<unsigned _Base>
struct _Digit<_Base, '8'> : _Digit_impl<_Base, 8>
{ };
template<unsigned _Base>
struct _Digit<_Base, '9'> : _Digit_impl<_Base, 9>
{ };
template<unsigned _Base>
struct _Digit<_Base, 'a'> : _Digit_impl<_Base, 0xa>
{ };
template<unsigned _Base>
struct _Digit<_Base, 'A'> : _Digit_impl<_Base, 0xa>
{ };
template<unsigned _Base>
struct _Digit<_Base, 'b'> : _Digit_impl<_Base, 0xb>
{ };
template<unsigned _Base>
struct _Digit<_Base, 'B'> : _Digit_impl<_Base, 0xb>
{ };
template<unsigned _Base>
struct _Digit<_Base, 'c'> : _Digit_impl<_Base, 0xc>
{ };
template<unsigned _Base>
struct _Digit<_Base, 'C'> : _Digit_impl<_Base, 0xc>
{ };
template<unsigned _Base>
struct _Digit<_Base, 'd'> : _Digit_impl<_Base, 0xd>
{ };
template<unsigned _Base>
struct _Digit<_Base, 'D'> : _Digit_impl<_Base, 0xd>
{ };
template<unsigned _Base>
struct _Digit<_Base, 'e'> : _Digit_impl<_Base, 0xe>
{ };
template<unsigned _Base>
struct _Digit<_Base, 'E'> : _Digit_impl<_Base, 0xe>
{ };
template<unsigned _Base>
struct _Digit<_Base, 'f'> : _Digit_impl<_Base, 0xf>
{ };
template<unsigned _Base>
struct _Digit<_Base, 'F'> : _Digit_impl<_Base, 0xf>
{ };
template<unsigned _Base>
struct _Digit<_Base, '\''> : integral_constant<unsigned, 0>
{
using __valid = false_type;
};
template<unsigned long long _Val>
using __ull_constant = integral_constant<unsigned long long, _Val>;
template<unsigned _Base, char _Dig, char... _Digs>
struct _Power_help
{
using __next = typename _Power_help<_Base, _Digs...>::type;
using __valid_digit = typename _Digit<_Base, _Dig>::__valid;
using type
= __ull_constant<__next::value * (__valid_digit{} ? _Base : 1ULL)>;
};
template<unsigned _Base, char _Dig>
struct _Power_help<_Base, _Dig>
{
using __valid_digit = typename _Digit<_Base, _Dig>::__valid;
using type = __ull_constant<__valid_digit::value>;
};
template<unsigned _Base, char... _Digs>
struct _Power : _Power_help<_Base, _Digs...>::type
{ };
template<unsigned _Base>
struct _Power<_Base> : __ull_constant<0>
{ };
template<unsigned _Base, unsigned long long _Pow, char _Dig, char... _Digs>
struct _Number_help
{
using __digit = _Digit<_Base, _Dig>;
using __valid_digit = typename __digit::__valid;
using __next = _Number_help<_Base,
__valid_digit::value ? _Pow / _Base : _Pow,
_Digs...>;
using type = __ull_constant<_Pow * __digit::value + __next::type::value>;
static_assert((type::value / _Pow) == __digit::value,
"integer literal does not fit in unsigned long long");
};
template<unsigned _Base, unsigned long long _Pow, char _Dig, char..._Digs>
struct _Number_help<_Base, _Pow, '\'', _Dig, _Digs...>
: _Number_help<_Base, _Pow, _Dig, _Digs...>
{ };
template<unsigned _Base, char _Dig>
struct _Number_help<_Base, 1ULL, _Dig>
{
using type = __ull_constant<_Digit<_Base, _Dig>::value>;
};
template<unsigned _Base, char... _Digs>
struct _Number
: _Number_help<_Base, _Power<_Base, _Digs...>::value, _Digs...>::type
{ };
template<unsigned _Base>
struct _Number<_Base>
: __ull_constant<0>
{ };
template<char... _Digs>
struct _Parse_int;
template<char... _Digs>
struct _Parse_int<'0', 'b', _Digs...>
: _Number<2U, _Digs...>::type
{ };
template<char... _Digs>
struct _Parse_int<'0', 'B', _Digs...>
: _Number<2U, _Digs...>::type
{ };
template<char... _Digs>
struct _Parse_int<'0', 'x', _Digs...>
: _Number<16U, _Digs...>::type
{ };
template<char... _Digs>
struct _Parse_int<'0', 'X', _Digs...>
: _Number<16U, _Digs...>::type
{ };
template<char... _Digs>
struct _Parse_int<'0', _Digs...>
: _Number<8U, _Digs...>::type
{ };
template<char... _Digs>
struct _Parse_int
: _Number<10U, _Digs...>::type
{ };
}
namespace __select_int
{
template<unsigned long long _Val, typename... _Ints>
struct _Select_int_base;
template<unsigned long long _Val, typename _IntType, typename... _Ints>
struct _Select_int_base<_Val, _IntType, _Ints...>
: __conditional_t<(_Val <= __gnu_cxx::__int_traits<_IntType>::__max),
integral_constant<_IntType, (_IntType)_Val>,
_Select_int_base<_Val, _Ints...>>
{ };
template<unsigned long long _Val>
struct _Select_int_base<_Val>
{ };
template<char... _Digs>
using _Select_int = typename _Select_int_base<
__parse_int::_Parse_int<_Digs...>::value,
unsigned char,
unsigned short,
unsigned int,
unsigned long,
unsigned long long
>::type;
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace filesystem { struct __file_clock; };
namespace chrono
{
template<typename _Rep, typename _Period = ratio<1>>
class duration;
template<typename _Clock, typename _Dur = typename _Clock::duration>
class time_point;
}
template<typename _CT, typename _Period1, typename _Period2, typename = void>
struct __duration_common_type
{ };
template<typename _CT, typename _Period1, typename _Period2>
struct __duration_common_type<_CT, _Period1, _Period2,
__void_t<typename _CT::type>>
{
private:
using __gcd_num = __static_gcd<_Period1::num, _Period2::num>;
using __gcd_den = __static_gcd<_Period1::den, _Period2::den>;
using __cr = typename _CT::type;
using __r = ratio<__gcd_num::value,
(_Period1::den / __gcd_den::value) * _Period2::den>;
public:
using type = chrono::duration<__cr, typename __r::type>;
};
template<typename _Rep1, typename _Period1, typename _Rep2, typename _Period2>
struct common_type<chrono::duration<_Rep1, _Period1>,
chrono::duration<_Rep2, _Period2>>
: __duration_common_type<common_type<_Rep1, _Rep2>,
typename _Period1::type,
typename _Period2::type>
{ };
template<typename _Rep, typename _Period>
struct common_type<chrono::duration<_Rep, _Period>,
chrono::duration<_Rep, _Period>>
{
using type = chrono::duration<typename common_type<_Rep>::type,
typename _Period::type>;
};
template<typename _Rep, typename _Period>
struct common_type<chrono::duration<_Rep, _Period>>
{
using type = chrono::duration<typename common_type<_Rep>::type,
typename _Period::type>;
};
template<typename _CT, typename _Clock, typename = void>
struct __timepoint_common_type
{ };
template<typename _CT, typename _Clock>
struct __timepoint_common_type<_CT, _Clock, __void_t<typename _CT::type>>
{
using type = chrono::time_point<_Clock, typename _CT::type>;
};
template<typename _Clock, typename _Duration1, typename _Duration2>
struct common_type<chrono::time_point<_Clock, _Duration1>,
chrono::time_point<_Clock, _Duration2>>
: __timepoint_common_type<common_type<_Duration1, _Duration2>, _Clock>
{ };
template<typename _Clock, typename _Duration>
struct common_type<chrono::time_point<_Clock, _Duration>,
chrono::time_point<_Clock, _Duration>>
{ using type = chrono::time_point<_Clock, _Duration>; };
template<typename _Clock, typename _Duration>
struct common_type<chrono::time_point<_Clock, _Duration>>
{ using type = chrono::time_point<_Clock, _Duration>; };
namespace chrono
{
template<typename _ToDur, typename _CF, typename _CR,
bool _NumIsOne = false, bool _DenIsOne = false>
struct __duration_cast_impl
{
template<typename _Rep, typename _Period>
static constexpr _ToDur
__cast(const duration<_Rep, _Period>& __d)
{
typedef typename _ToDur::rep __to_rep;
return _ToDur(static_cast<__to_rep>(static_cast<_CR>(__d.count())
* static_cast<_CR>(_CF::num)
/ static_cast<_CR>(_CF::den)));
}
};
template<typename _ToDur, typename _CF, typename _CR>
struct __duration_cast_impl<_ToDur, _CF, _CR, true, true>
{
template<typename _Rep, typename _Period>
static constexpr _ToDur
__cast(const duration<_Rep, _Period>& __d)
{
typedef typename _ToDur::rep __to_rep;
return _ToDur(static_cast<__to_rep>(__d.count()));
}
};
template<typename _ToDur, typename _CF, typename _CR>
struct __duration_cast_impl<_ToDur, _CF, _CR, true, false>
{
template<typename _Rep, typename _Period>
static constexpr _ToDur
__cast(const duration<_Rep, _Period>& __d)
{
typedef typename _ToDur::rep __to_rep;
return _ToDur(static_cast<__to_rep>(
static_cast<_CR>(__d.count()) / static_cast<_CR>(_CF::den)));
}
};
template<typename _ToDur, typename _CF, typename _CR>
struct __duration_cast_impl<_ToDur, _CF, _CR, false, true>
{
template<typename _Rep, typename _Period>
static constexpr _ToDur
__cast(const duration<_Rep, _Period>& __d)
{
typedef typename _ToDur::rep __to_rep;
return _ToDur(static_cast<__to_rep>(
static_cast<_CR>(__d.count()) * static_cast<_CR>(_CF::num)));
}
};
template<typename _Tp>
struct __is_duration
: std::false_type
{ };
template<typename _Rep, typename _Period>
struct __is_duration<duration<_Rep, _Period>>
: std::true_type
{ };
template<typename _Tp>
using __enable_if_is_duration
= typename enable_if<__is_duration<_Tp>::value, _Tp>::type;
template<typename _Tp>
using __disable_if_is_duration
= typename enable_if<!__is_duration<_Tp>::value, _Tp>::type;
template<typename _Tp>
inline constexpr bool __is_duration_v = false;
template<typename _Rep, typename _Period>
inline constexpr bool __is_duration_v<duration<_Rep, _Period>> = true;
template<typename _Tp>
inline constexpr bool __is_time_point_v = false;
template<typename _Clock, typename _Dur>
inline constexpr bool __is_time_point_v<time_point<_Clock, _Dur>> = true;
template<typename _ToDur, typename _Rep, typename _Period>
[[__nodiscard__]]
constexpr __enable_if_is_duration<_ToDur>
duration_cast(const duration<_Rep, _Period>& __d)
{
if constexpr (is_same_v<_ToDur, duration<_Rep, _Period>>)
return __d;
else
{
using __to_period = typename _ToDur::period;
using __to_rep = typename _ToDur::rep;
using __cf = ratio_divide<_Period, __to_period>;
using __cr = typename common_type<__to_rep, _Rep, intmax_t>::type;
using __dc = __duration_cast_impl<_ToDur, __cf, __cr,
__cf::num == 1, __cf::den == 1>;
return __dc::__cast(__d);
}
}
template<typename _Rep>
struct treat_as_floating_point
: is_floating_point<_Rep>
{ };
template <typename _Rep>
inline constexpr bool treat_as_floating_point_v =
treat_as_floating_point<_Rep>::value;
template<>
inline constexpr bool treat_as_floating_point_v<int> = false;
template<>
inline constexpr bool treat_as_floating_point_v<long> = false;
template<>
inline constexpr bool treat_as_floating_point_v<long long> = false;
template<>
inline constexpr bool treat_as_floating_point_v<float> = true;
template<>
inline constexpr bool treat_as_floating_point_v<double> = true;
template<>
inline constexpr bool treat_as_floating_point_v<long double> = true;
template<typename _ToDur, typename _Rep, typename _Period>
[[nodiscard]] constexpr __enable_if_is_duration<_ToDur>
floor(const duration<_Rep, _Period>& __d)
{
auto __to = chrono::duration_cast<_ToDur>(__d);
if (__to > __d)
return __to - _ToDur{1};
return __to;
}
template<typename _ToDur, typename _Rep, typename _Period>
[[nodiscard]] constexpr __enable_if_is_duration<_ToDur>
ceil(const duration<_Rep, _Period>& __d)
{
auto __to = chrono::duration_cast<_ToDur>(__d);
if (__to < __d)
return __to + _ToDur{1};
return __to;
}
template <typename _ToDur, typename _Rep, typename _Period>
[[nodiscard]] constexpr
enable_if_t<
__and_<__is_duration<_ToDur>,
__not_<treat_as_floating_point<typename _ToDur::rep>>>::value,
_ToDur>
round(const duration<_Rep, _Period>& __d)
{
_ToDur __t0 = chrono::floor<_ToDur>(__d);
_ToDur __t1 = __t0 + _ToDur{1};
auto __diff0 = __d - __t0;
auto __diff1 = __t1 - __d;
if (__diff0 == __diff1)
{
if (__t0.count() & 1)
return __t1;
return __t0;
}
else if (__diff0 < __diff1)
return __t0;
return __t1;
}
template<typename _Rep, typename _Period>
[[nodiscard]] constexpr
enable_if_t<numeric_limits<_Rep>::is_signed, duration<_Rep, _Period>>
abs(duration<_Rep, _Period> __d)
{
if (__d >= __d.zero())
return __d;
return -__d;
}
namespace __detail { using chrono::ceil; }
template<typename _Rep>
struct duration_values
{
static constexpr _Rep
zero() noexcept
{ return _Rep(0); }
static constexpr _Rep
max() noexcept
{ return numeric_limits<_Rep>::max(); }
static constexpr _Rep
min() noexcept
{ return numeric_limits<_Rep>::lowest(); }
};
template<typename _Rep, typename _Period>
class duration
{
static_assert(!__is_duration<_Rep>::value,
"rep cannot be a std::chrono::duration");
static_assert(__is_ratio<_Period>::value,
"period must be a specialization of std::ratio");
static_assert(_Period::num > 0, "period must be positive");
template<typename _Rep2>
using __is_float = treat_as_floating_point<_Rep2>;
static constexpr intmax_t
_S_gcd(intmax_t __m, intmax_t __n) noexcept
{
do
{
intmax_t __rem = __m % __n;
__m = __n;
__n = __rem;
}
while (__n != 0);
return __m;
}
template<typename _R1, typename _R2,
intmax_t __gcd1 = _S_gcd(_R1::num, _R2::num),
intmax_t __gcd2 = _S_gcd(_R1::den, _R2::den)>
using __divide = ratio<(_R1::num / __gcd1) * (_R2::den / __gcd2),
(_R1::den / __gcd2) * (_R2::num / __gcd1)>;
template<typename _Period2>
using __is_harmonic
= __bool_constant<__divide<_Period2, _Period>::den == 1>;
public:
using rep = _Rep;
using period = typename _Period::type;
constexpr duration() = default;
duration(const duration&) = default;
template<typename _Rep2, typename = _Require<
is_convertible<const _Rep2&, rep>,
__or_<__is_float<rep>, __not_<__is_float<_Rep2>>>>>
constexpr explicit duration(const _Rep2& __rep)
: __r(static_cast<rep>(__rep)) { }
template<typename _Rep2, typename _Period2, typename = _Require<
is_convertible<const _Rep2&, rep>,
__or_<__is_float<rep>,
__and_<__is_harmonic<_Period2>,
__not_<__is_float<_Rep2>>>>>>
constexpr duration(const duration<_Rep2, _Period2>& __d)
: __r(duration_cast<duration>(__d).count()) { }
~duration() = default;
duration& operator=(const duration&) = default;
constexpr rep
count() const
{ return __r; }
constexpr duration<typename common_type<rep>::type, period>
operator+() const
{ return duration<typename common_type<rep>::type, period>(__r); }
constexpr duration<typename common_type<rep>::type, period>
operator-() const
{ return duration<typename common_type<rep>::type, period>(-__r); }
constexpr duration&
operator++()
{
++__r;
return *this;
}
constexpr duration
operator++(int)
{ return duration(__r++); }
constexpr duration&
operator--()
{
--__r;
return *this;
}
constexpr duration
operator--(int)
{ return duration(__r--); }
constexpr duration&
operator+=(const duration& __d)
{
__r += __d.count();
return *this;
}
constexpr duration&
operator-=(const duration& __d)
{
__r -= __d.count();
return *this;
}
constexpr duration&
operator*=(const rep& __rhs)
{
__r *= __rhs;
return *this;
}
constexpr duration&
operator/=(const rep& __rhs)
{
__r /= __rhs;
return *this;
}
template<typename _Rep2 = rep>
constexpr
__enable_if_t<!treat_as_floating_point<_Rep2>::value, duration&>
operator%=(const rep& __rhs)
{
__r %= __rhs;
return *this;
}
template<typename _Rep2 = rep>
constexpr
__enable_if_t<!treat_as_floating_point<_Rep2>::value, duration&>
operator%=(const duration& __d)
{
__r %= __d.count();
return *this;
}
static constexpr duration
zero() noexcept
{ return duration(duration_values<rep>::zero()); }
static constexpr duration
min() noexcept
{ return duration(duration_values<rep>::min()); }
static constexpr duration
max() noexcept
{ return duration(duration_values<rep>::max()); }
private:
rep __r;
};
template<typename _Rep1, typename _Period1,
typename _Rep2, typename _Period2>
constexpr typename common_type<duration<_Rep1, _Period1>,
duration<_Rep2, _Period2>>::type
operator+(const duration<_Rep1, _Period1>& __lhs,
const duration<_Rep2, _Period2>& __rhs)
{
typedef duration<_Rep1, _Period1> __dur1;
typedef duration<_Rep2, _Period2> __dur2;
typedef typename common_type<__dur1,__dur2>::type __cd;
return __cd(__cd(__lhs).count() + __cd(__rhs).count());
}
template<typename _Rep1, typename _Period1,
typename _Rep2, typename _Period2>
constexpr typename common_type<duration<_Rep1, _Period1>,
duration<_Rep2, _Period2>>::type
operator-(const duration<_Rep1, _Period1>& __lhs,
const duration<_Rep2, _Period2>& __rhs)
{
typedef duration<_Rep1, _Period1> __dur1;
typedef duration<_Rep2, _Period2> __dur2;
typedef typename common_type<__dur1,__dur2>::type __cd;
return __cd(__cd(__lhs).count() - __cd(__rhs).count());
}
template<typename _Rep1, typename _Rep2,
typename _CRep = typename common_type<_Rep1, _Rep2>::type>
using __common_rep_t = typename
enable_if<is_convertible<const _Rep2&, _CRep>::value, _CRep>::type;
template<typename _Rep1, typename _Period, typename _Rep2>
constexpr duration<__common_rep_t<_Rep1, _Rep2>, _Period>
operator*(const duration<_Rep1, _Period>& __d, const _Rep2& __s)
{
typedef duration<typename common_type<_Rep1, _Rep2>::type, _Period>
__cd;
return __cd(__cd(__d).count() * __s);
}
template<typename _Rep1, typename _Rep2, typename _Period>
constexpr duration<__common_rep_t<_Rep2, _Rep1>, _Period>
operator*(const _Rep1& __s, const duration<_Rep2, _Period>& __d)
{ return __d * __s; }
template<typename _Rep1, typename _Period, typename _Rep2>
constexpr
duration<__common_rep_t<_Rep1, __disable_if_is_duration<_Rep2>>, _Period>
operator/(const duration<_Rep1, _Period>& __d, const _Rep2& __s)
{
typedef duration<typename common_type<_Rep1, _Rep2>::type, _Period>
__cd;
return __cd(__cd(__d).count() / __s);
}
template<typename _Rep1, typename _Period1,
typename _Rep2, typename _Period2>
constexpr typename common_type<_Rep1, _Rep2>::type
operator/(const duration<_Rep1, _Period1>& __lhs,
const duration<_Rep2, _Period2>& __rhs)
{
typedef duration<_Rep1, _Period1> __dur1;
typedef duration<_Rep2, _Period2> __dur2;
typedef typename common_type<__dur1,__dur2>::type __cd;
return __cd(__lhs).count() / __cd(__rhs).count();
}
template<typename _Rep1, typename _Period, typename _Rep2>
constexpr
duration<__common_rep_t<_Rep1, __disable_if_is_duration<_Rep2>>, _Period>
operator%(const duration<_Rep1, _Period>& __d, const _Rep2& __s)
{
typedef duration<typename common_type<_Rep1, _Rep2>::type, _Period>
__cd;
return __cd(__cd(__d).count() % __s);
}
template<typename _Rep1, typename _Period1,
typename _Rep2, typename _Period2>
constexpr typename common_type<duration<_Rep1, _Period1>,
duration<_Rep2, _Period2>>::type
operator%(const duration<_Rep1, _Period1>& __lhs,
const duration<_Rep2, _Period2>& __rhs)
{
typedef duration<_Rep1, _Period1> __dur1;
typedef duration<_Rep2, _Period2> __dur2;
typedef typename common_type<__dur1,__dur2>::type __cd;
return __cd(__cd(__lhs).count() % __cd(__rhs).count());
}
template<typename _Rep1, typename _Period1,
typename _Rep2, typename _Period2>
constexpr bool
operator==(const duration<_Rep1, _Period1>& __lhs,
const duration<_Rep2, _Period2>& __rhs)
{
typedef duration<_Rep1, _Period1> __dur1;
typedef duration<_Rep2, _Period2> __dur2;
typedef typename common_type<__dur1,__dur2>::type __ct;
return __ct(__lhs).count() == __ct(__rhs).count();
}
template<typename _Rep1, typename _Period1,
typename _Rep2, typename _Period2>
constexpr bool
operator<(const duration<_Rep1, _Period1>& __lhs,
const duration<_Rep2, _Period2>& __rhs)
{
typedef duration<_Rep1, _Period1> __dur1;
typedef duration<_Rep2, _Period2> __dur2;
typedef typename common_type<__dur1,__dur2>::type __ct;
return __ct(__lhs).count() < __ct(__rhs).count();
}
template<typename _Rep1, typename _Period1,
typename _Rep2, typename _Period2>
constexpr bool
operator!=(const duration<_Rep1, _Period1>& __lhs,
const duration<_Rep2, _Period2>& __rhs)
{ return !(__lhs == __rhs); }
template<typename _Rep1, typename _Period1,
typename _Rep2, typename _Period2>
constexpr bool
operator<=(const duration<_Rep1, _Period1>& __lhs,
const duration<_Rep2, _Period2>& __rhs)
{ return !(__rhs < __lhs); }
template<typename _Rep1, typename _Period1,
typename _Rep2, typename _Period2>
constexpr bool
operator>(const duration<_Rep1, _Period1>& __lhs,
const duration<_Rep2, _Period2>& __rhs)
{ return __rhs < __lhs; }
template<typename _Rep1, typename _Period1,
typename _Rep2, typename _Period2>
constexpr bool
operator>=(const duration<_Rep1, _Period1>& __lhs,
const duration<_Rep2, _Period2>& __rhs)
{ return !(__lhs < __rhs); }
using nanoseconds = duration<int64_t, nano>;
using microseconds = duration<int64_t, micro>;
using milliseconds = duration<int64_t, milli>;
using seconds = duration<int64_t>;
using minutes = duration<int64_t, ratio< 60>>;
using hours = duration<int64_t, ratio<3600>>;
template<typename _Clock, typename _Dur>
class time_point
{
static_assert(__is_duration<_Dur>::value,
"duration must be a specialization of std::chrono::duration");
public:
typedef _Clock clock;
typedef _Dur duration;
typedef typename duration::rep rep;
typedef typename duration::period period;
constexpr time_point() : __d(duration::zero())
{ }
constexpr explicit time_point(const duration& __dur)
: __d(__dur)
{ }
template<typename _Dur2,
typename = _Require<is_convertible<_Dur2, _Dur>>>
constexpr time_point(const time_point<clock, _Dur2>& __t)
: __d(__t.time_since_epoch())
{ }
constexpr duration
time_since_epoch() const
{ return __d; }
constexpr time_point&
operator+=(const duration& __dur)
{
__d += __dur;
return *this;
}
constexpr time_point&
operator-=(const duration& __dur)
{
__d -= __dur;
return *this;
}
static constexpr time_point
min() noexcept
{ return time_point(duration::min()); }
static constexpr time_point
max() noexcept
{ return time_point(duration::max()); }
private:
duration __d;
};
template<typename _ToDur, typename _Clock, typename _Dur>
[[__nodiscard__]] constexpr
__enable_if_t<__is_duration<_ToDur>::value, time_point<_Clock, _ToDur>>
time_point_cast(const time_point<_Clock, _Dur>& __t)
{
typedef time_point<_Clock, _ToDur> __time_point;
return __time_point(duration_cast<_ToDur>(__t.time_since_epoch()));
}
template<typename _ToDur, typename _Clock, typename _Dur>
[[nodiscard]] constexpr
enable_if_t<__is_duration_v<_ToDur>, time_point<_Clock, _ToDur>>
floor(const time_point<_Clock, _Dur>& __tp)
{
return time_point<_Clock, _ToDur>{
chrono::floor<_ToDur>(__tp.time_since_epoch())};
}
template<typename _ToDur, typename _Clock, typename _Dur>
[[nodiscard]] constexpr
enable_if_t<__is_duration_v<_ToDur>, time_point<_Clock, _ToDur>>
ceil(const time_point<_Clock, _Dur>& __tp)
{
return time_point<_Clock, _ToDur>{
chrono::ceil<_ToDur>(__tp.time_since_epoch())};
}
template<typename _ToDur, typename _Clock, typename _Dur>
[[nodiscard]] constexpr
enable_if_t<__is_duration_v<_ToDur>
&& !treat_as_floating_point_v<typename _ToDur::rep>,
time_point<_Clock, _ToDur>>
round(const time_point<_Clock, _Dur>& __tp)
{
return time_point<_Clock, _ToDur>{
chrono::round<_ToDur>(__tp.time_since_epoch())};
}
template<typename _Clock, typename _Dur1,
typename _Rep2, typename _Period2>
constexpr time_point<_Clock,
typename common_type<_Dur1, duration<_Rep2, _Period2>>::type>
operator+(const time_point<_Clock, _Dur1>& __lhs,
const duration<_Rep2, _Period2>& __rhs)
{
typedef duration<_Rep2, _Period2> __dur2;
typedef typename common_type<_Dur1,__dur2>::type __ct;
typedef time_point<_Clock, __ct> __time_point;
return __time_point(__lhs.time_since_epoch() + __rhs);
}
template<typename _Rep1, typename _Period1,
typename _Clock, typename _Dur2>
constexpr time_point<_Clock,
typename common_type<duration<_Rep1, _Period1>, _Dur2>::type>
operator+(const duration<_Rep1, _Period1>& __lhs,
const time_point<_Clock, _Dur2>& __rhs)
{
typedef duration<_Rep1, _Period1> __dur1;
typedef typename common_type<__dur1,_Dur2>::type __ct;
typedef time_point<_Clock, __ct> __time_point;
return __time_point(__rhs.time_since_epoch() + __lhs);
}
template<typename _Clock, typename _Dur1,
typename _Rep2, typename _Period2>
constexpr time_point<_Clock,
typename common_type<_Dur1, duration<_Rep2, _Period2>>::type>
operator-(const time_point<_Clock, _Dur1>& __lhs,
const duration<_Rep2, _Period2>& __rhs)
{
typedef duration<_Rep2, _Period2> __dur2;
typedef typename common_type<_Dur1,__dur2>::type __ct;
typedef time_point<_Clock, __ct> __time_point;
return __time_point(__lhs.time_since_epoch() -__rhs);
}
template<typename _Clock, typename _Dur1, typename _Dur2>
constexpr typename common_type<_Dur1, _Dur2>::type
operator-(const time_point<_Clock, _Dur1>& __lhs,
const time_point<_Clock, _Dur2>& __rhs)
{ return __lhs.time_since_epoch() - __rhs.time_since_epoch(); }
template<typename _Clock, typename _Dur1, typename _Dur2>
constexpr bool
operator==(const time_point<_Clock, _Dur1>& __lhs,
const time_point<_Clock, _Dur2>& __rhs)
{ return __lhs.time_since_epoch() == __rhs.time_since_epoch(); }
template<typename _Clock, typename _Dur1, typename _Dur2>
constexpr bool
operator!=(const time_point<_Clock, _Dur1>& __lhs,
const time_point<_Clock, _Dur2>& __rhs)
{ return !(__lhs == __rhs); }
template<typename _Clock, typename _Dur1, typename _Dur2>
constexpr bool
operator<(const time_point<_Clock, _Dur1>& __lhs,
const time_point<_Clock, _Dur2>& __rhs)
{ return __lhs.time_since_epoch() < __rhs.time_since_epoch(); }
template<typename _Clock, typename _Dur1, typename _Dur2>
constexpr bool
operator<=(const time_point<_Clock, _Dur1>& __lhs,
const time_point<_Clock, _Dur2>& __rhs)
{ return !(__rhs < __lhs); }
template<typename _Clock, typename _Dur1, typename _Dur2>
constexpr bool
operator>(const time_point<_Clock, _Dur1>& __lhs,
const time_point<_Clock, _Dur2>& __rhs)
{ return __rhs < __lhs; }
template<typename _Clock, typename _Dur1, typename _Dur2>
constexpr bool
operator>=(const time_point<_Clock, _Dur1>& __lhs,
const time_point<_Clock, _Dur2>& __rhs)
{ return !(__lhs < __rhs); }
inline namespace _V2 {
struct system_clock
{
typedef chrono::nanoseconds duration;
typedef duration::rep rep;
typedef duration::period period;
typedef chrono::time_point<system_clock, duration> time_point;
static_assert(system_clock::duration::min()
< system_clock::duration::zero(),
"a clock's minimum duration cannot be less than its epoch");
static constexpr bool is_steady = false;
static time_point
now() noexcept;
static std::time_t
to_time_t(const time_point& __t) noexcept
{
return std::time_t(duration_cast<chrono::seconds>
(__t.time_since_epoch()).count());
}
static time_point
from_time_t(std::time_t __t) noexcept
{
typedef chrono::time_point<system_clock, seconds> __from;
return time_point_cast<system_clock::duration>
(__from(chrono::seconds(__t)));
}
};
struct steady_clock
{
typedef chrono::nanoseconds duration;
typedef duration::rep rep;
typedef duration::period period;
typedef chrono::time_point<steady_clock, duration> time_point;
static constexpr bool is_steady = true;
static time_point
now() noexcept;
};
using high_resolution_clock = system_clock;
}
}
inline namespace literals
{
inline namespace chrono_literals
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wliteral-suffix"
template<typename _Dur, char... _Digits>
constexpr _Dur __check_overflow()
{
using _Val = __parse_int::_Parse_int<_Digits...>;
constexpr typename _Dur::rep __repval = _Val::value;
static_assert(__repval >= 0 && __repval == _Val::value,
"literal value cannot be represented by duration type");
return _Dur(__repval);
}
constexpr chrono::duration<long double, ratio<3600,1>>
operator""h(long double __hours)
{ return chrono::duration<long double, ratio<3600,1>>{__hours}; }
template <char... _Digits>
constexpr chrono::hours
operator""h()
{ return __check_overflow<chrono::hours, _Digits...>(); }
constexpr chrono::duration<long double, ratio<60,1>>
operator""min(long double __mins)
{ return chrono::duration<long double, ratio<60,1>>{__mins}; }
template <char... _Digits>
constexpr chrono::minutes
operator""min()
{ return __check_overflow<chrono::minutes, _Digits...>(); }
constexpr chrono::duration<long double>
operator""s(long double __secs)
{ return chrono::duration<long double>{__secs}; }
template <char... _Digits>
constexpr chrono::seconds
operator""s()
{ return __check_overflow<chrono::seconds, _Digits...>(); }
constexpr chrono::duration<long double, milli>
operator""ms(long double __msecs)
{ return chrono::duration<long double, milli>{__msecs}; }
template <char... _Digits>
constexpr chrono::milliseconds
operator""ms()
{ return __check_overflow<chrono::milliseconds, _Digits...>(); }
constexpr chrono::duration<long double, micro>
operator""us(long double __usecs)
{ return chrono::duration<long double, micro>{__usecs}; }
template <char... _Digits>
constexpr chrono::microseconds
operator""us()
{ return __check_overflow<chrono::microseconds, _Digits...>(); }
constexpr chrono::duration<long double, nano>
operator""ns(long double __nsecs)
{ return chrono::duration<long double, nano>{__nsecs}; }
template <char... _Digits>
constexpr chrono::nanoseconds
operator""ns()
{ return __check_overflow<chrono::nanoseconds, _Digits...>(); }
#pragma GCC diagnostic pop
}
}
namespace chrono
{
using namespace literals::chrono_literals;
}
namespace filesystem
{
struct __file_clock
{
using duration = chrono::nanoseconds;
using rep = duration::rep;
using period = duration::period;
using time_point = chrono::time_point<__file_clock>;
static constexpr bool is_steady = false;
static time_point
now() noexcept
{ return _S_from_sys(chrono::system_clock::now()); }
private:
using __sys_clock = chrono::system_clock;
static constexpr chrono::seconds _S_epoch_diff{6437664000};
protected:
template<typename _Dur>
static
chrono::time_point<__file_clock, _Dur>
_S_from_sys(const chrono::time_point<__sys_clock, _Dur>& __t) noexcept
{
using __file_time = chrono::time_point<__file_clock, _Dur>;
return __file_time{__t.time_since_epoch()} - _S_epoch_diff;
}
template<typename _Dur>
static
chrono::time_point<__sys_clock, _Dur>
_S_to_sys(const chrono::time_point<__file_clock, _Dur>& __t) noexcept
{
using __sys_time = chrono::time_point<__sys_clock, _Dur>;
return __sys_time{__t.time_since_epoch()} + _S_epoch_diff;
}
};
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
class __mutex_base
{
protected:
typedef __gthread_mutex_t __native_type;
__native_type _M_mutex = { { 0, 0, 0, PTHREAD_MUTEX_TIMED_NP, 0, { { 0, 0 } } } };
constexpr __mutex_base() noexcept = default;
__mutex_base(const __mutex_base&) = delete;
__mutex_base& operator=(const __mutex_base&) = delete;
};
class mutex : private __mutex_base
{
public:
typedef __native_type* native_handle_type;
constexpr
mutex() noexcept = default;
~mutex() = default;
mutex(const mutex&) = delete;
mutex& operator=(const mutex&) = delete;
void
lock()
{
int __e = __gthread_mutex_lock(&_M_mutex);
if (__e)
__throw_system_error(__e);
}
[[__nodiscard__]]
bool
try_lock() noexcept
{
return !__gthread_mutex_trylock(&_M_mutex);
}
void
unlock()
{
__gthread_mutex_unlock(&_M_mutex);
}
native_handle_type
native_handle() noexcept
{ return &_M_mutex; }
};
class __condvar
{
using timespec = __gthread_time_t;
public:
__condvar() noexcept
{
}
~__condvar()
{
int __e __attribute__((__unused__)) = __gthread_cond_destroy(&_M_cond);
do { if (std::__is_constant_evaluated() && !bool(__e != 16)) __builtin_unreachable(); } while (false);
}
__condvar(const __condvar&) = delete;
__condvar& operator=(const __condvar&) = delete;
__gthread_cond_t* native_handle() noexcept { return &_M_cond; }
void
wait(mutex& __m)
{
int __e __attribute__((__unused__))
= __gthread_cond_wait(&_M_cond, __m.native_handle());
do { if (std::__is_constant_evaluated() && !bool(__e == 0)) __builtin_unreachable(); } while (false);
}
void
wait_until(mutex& __m, timespec& __abs_time)
{
__gthread_cond_timedwait(&_M_cond, __m.native_handle(), &__abs_time);
}
void
wait_until(mutex& __m, clockid_t __clock, timespec& __abs_time)
{
pthread_cond_clockwait(&_M_cond, __m.native_handle(), __clock,
&__abs_time);
}
void
notify_one() noexcept
{
int __e __attribute__((__unused__)) = __gthread_cond_signal(&_M_cond);
do { if (std::__is_constant_evaluated() && !bool(__e == 0)) __builtin_unreachable(); } while (false);
}
void
notify_all() noexcept
{
int __e __attribute__((__unused__)) = __gthread_cond_broadcast(&_M_cond);
do { if (std::__is_constant_evaluated() && !bool(__e == 0)) __builtin_unreachable(); } while (false);
}
protected:
__gthread_cond_t _M_cond = { { {0}, {0}, {0, 0}, {0, 0}, 0, 0, {0, 0} } };
};
struct defer_lock_t { explicit defer_lock_t() = default; };
struct try_to_lock_t { explicit try_to_lock_t() = default; };
struct adopt_lock_t { explicit adopt_lock_t() = default; };
inline constexpr defer_lock_t defer_lock { };
inline constexpr try_to_lock_t try_to_lock { };
inline constexpr adopt_lock_t adopt_lock { };
template<typename _Mutex>
class lock_guard
{
public:
typedef _Mutex mutex_type;
explicit lock_guard(mutex_type& __m) : _M_device(__m)
{ _M_device.lock(); }
lock_guard(mutex_type& __m, adopt_lock_t) noexcept : _M_device(__m)
{ }
~lock_guard()
{ _M_device.unlock(); }
lock_guard(const lock_guard&) = delete;
lock_guard& operator=(const lock_guard&) = delete;
private:
mutex_type& _M_device;
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Mutex>
class unique_lock
{
public:
typedef _Mutex mutex_type;
unique_lock() noexcept
: _M_device(0), _M_owns(false)
{ }
explicit unique_lock(mutex_type& __m)
: _M_device(std::__addressof(__m)), _M_owns(false)
{
lock();
_M_owns = true;
}
unique_lock(mutex_type& __m, defer_lock_t) noexcept
: _M_device(std::__addressof(__m)), _M_owns(false)
{ }
unique_lock(mutex_type& __m, try_to_lock_t)
: _M_device(std::__addressof(__m)), _M_owns(_M_device->try_lock())
{ }
unique_lock(mutex_type& __m, adopt_lock_t) noexcept
: _M_device(std::__addressof(__m)), _M_owns(true)
{
}
template<typename _Clock, typename _Duration>
unique_lock(mutex_type& __m,
const chrono::time_point<_Clock, _Duration>& __atime)
: _M_device(std::__addressof(__m)),
_M_owns(_M_device->try_lock_until(__atime))
{ }
template<typename _Rep, typename _Period>
unique_lock(mutex_type& __m,
const chrono::duration<_Rep, _Period>& __rtime)
: _M_device(std::__addressof(__m)),
_M_owns(_M_device->try_lock_for(__rtime))
{ }
~unique_lock()
{
if (_M_owns)
unlock();
}
unique_lock(const unique_lock&) = delete;
unique_lock& operator=(const unique_lock&) = delete;
unique_lock(unique_lock&& __u) noexcept
: _M_device(__u._M_device), _M_owns(__u._M_owns)
{
__u._M_device = 0;
__u._M_owns = false;
}
unique_lock& operator=(unique_lock&& __u) noexcept
{
if(_M_owns)
unlock();
unique_lock(std::move(__u)).swap(*this);
__u._M_device = 0;
__u._M_owns = false;
return *this;
}
void
lock()
{
if (!_M_device)
__throw_system_error(int(errc::operation_not_permitted));
else if (_M_owns)
__throw_system_error(int(errc::resource_deadlock_would_occur));
else
{
_M_device->lock();
_M_owns = true;
}
}
[[__nodiscard__]]
bool
try_lock()
{
if (!_M_device)
__throw_system_error(int(errc::operation_not_permitted));
else if (_M_owns)
__throw_system_error(int(errc::resource_deadlock_would_occur));
else
{
_M_owns = _M_device->try_lock();
return _M_owns;
}
}
template<typename _Clock, typename _Duration>
[[__nodiscard__]]
bool
try_lock_until(const chrono::time_point<_Clock, _Duration>& __atime)
{
if (!_M_device)
__throw_system_error(int(errc::operation_not_permitted));
else if (_M_owns)
__throw_system_error(int(errc::resource_deadlock_would_occur));
else
{
_M_owns = _M_device->try_lock_until(__atime);
return _M_owns;
}
}
template<typename _Rep, typename _Period>
[[__nodiscard__]]
bool
try_lock_for(const chrono::duration<_Rep, _Period>& __rtime)
{
if (!_M_device)
__throw_system_error(int(errc::operation_not_permitted));
else if (_M_owns)
__throw_system_error(int(errc::resource_deadlock_would_occur));
else
{
_M_owns = _M_device->try_lock_for(__rtime);
return _M_owns;
}
}
void
unlock()
{
if (!_M_owns)
__throw_system_error(int(errc::operation_not_permitted));
else if (_M_device)
{
_M_device->unlock();
_M_owns = false;
}
}
void
swap(unique_lock& __u) noexcept
{
std::swap(_M_device, __u._M_device);
std::swap(_M_owns, __u._M_owns);
}
mutex_type*
release() noexcept
{
mutex_type* __ret = _M_device;
_M_device = 0;
_M_owns = false;
return __ret;
}
[[__nodiscard__]]
bool
owns_lock() const noexcept
{ return _M_owns; }
explicit operator bool() const noexcept
{ return owns_lock(); }
[[__nodiscard__]]
mutex_type*
mutex() const noexcept
{ return _M_device; }
private:
mutex_type* _M_device;
bool _M_owns;
};
template<typename _Mutex>
inline void
swap(unique_lock<_Mutex>& __x, unique_lock<_Mutex>& __y) noexcept
{ __x.swap(__y); }
}
namespace std __attribute__ ((__visibility__ ("default")))
{
enum class cv_status { no_timeout, timeout };
class condition_variable
{
using steady_clock = chrono::steady_clock;
using system_clock = chrono::system_clock;
using __clock_t = steady_clock;
__condvar _M_cond;
public:
typedef __gthread_cond_t* native_handle_type;
condition_variable() noexcept;
~condition_variable() noexcept;
condition_variable(const condition_variable&) = delete;
condition_variable& operator=(const condition_variable&) = delete;
void
notify_one() noexcept;
void
notify_all() noexcept;
void
wait(unique_lock<mutex>& __lock);
template<typename _Predicate>
void
wait(unique_lock<mutex>& __lock, _Predicate __p)
{
while (!__p())
wait(__lock);
}
template<typename _Duration>
cv_status
wait_until(unique_lock<mutex>& __lock,
const chrono::time_point<steady_clock, _Duration>& __atime)
{ return __wait_until_impl(__lock, __atime); }
template<typename _Duration>
cv_status
wait_until(unique_lock<mutex>& __lock,
const chrono::time_point<system_clock, _Duration>& __atime)
{ return __wait_until_impl(__lock, __atime); }
template<typename _Clock, typename _Duration>
cv_status
wait_until(unique_lock<mutex>& __lock,
const chrono::time_point<_Clock, _Duration>& __atime)
{
using __s_dur = typename __clock_t::duration;
const typename _Clock::time_point __c_entry = _Clock::now();
const __clock_t::time_point __s_entry = __clock_t::now();
const auto __delta = __atime - __c_entry;
const auto __s_atime = __s_entry +
chrono::__detail::ceil<__s_dur>(__delta);
if (__wait_until_impl(__lock, __s_atime) == cv_status::no_timeout)
return cv_status::no_timeout;
if (_Clock::now() < __atime)
return cv_status::no_timeout;
return cv_status::timeout;
}
template<typename _Clock, typename _Duration, typename _Predicate>
bool
wait_until(unique_lock<mutex>& __lock,
const chrono::time_point<_Clock, _Duration>& __atime,
_Predicate __p)
{
while (!__p())
if (wait_until(__lock, __atime) == cv_status::timeout)
return __p();
return true;
}
template<typename _Rep, typename _Period>
cv_status
wait_for(unique_lock<mutex>& __lock,
const chrono::duration<_Rep, _Period>& __rtime)
{
using __dur = typename steady_clock::duration;
return wait_until(__lock,
steady_clock::now() +
chrono::__detail::ceil<__dur>(__rtime));
}
template<typename _Rep, typename _Period, typename _Predicate>
bool
wait_for(unique_lock<mutex>& __lock,
const chrono::duration<_Rep, _Period>& __rtime,
_Predicate __p)
{
using __dur = typename steady_clock::duration;
return wait_until(__lock,
steady_clock::now() +
chrono::__detail::ceil<__dur>(__rtime),
std::move(__p));
}
native_handle_type
native_handle()
{ return _M_cond.native_handle(); }
private:
template<typename _Dur>
cv_status
__wait_until_impl(unique_lock<mutex>& __lock,
const chrono::time_point<steady_clock, _Dur>& __atime)
{
auto __s = chrono::time_point_cast<chrono::seconds>(__atime);
auto __ns = chrono::duration_cast<chrono::nanoseconds>(__atime - __s);
__gthread_time_t __ts =
{
static_cast<std::time_t>(__s.time_since_epoch().count()),
static_cast<long>(__ns.count())
};
_M_cond.wait_until(*__lock.mutex(), 1, __ts);
return (steady_clock::now() < __atime
? cv_status::no_timeout : cv_status::timeout);
}
template<typename _Dur>
cv_status
__wait_until_impl(unique_lock<mutex>& __lock,
const chrono::time_point<system_clock, _Dur>& __atime)
{
auto __s = chrono::time_point_cast<chrono::seconds>(__atime);
auto __ns = chrono::duration_cast<chrono::nanoseconds>(__atime - __s);
__gthread_time_t __ts =
{
static_cast<std::time_t>(__s.time_since_epoch().count()),
static_cast<long>(__ns.count())
};
_M_cond.wait_until(*__lock.mutex(), __ts);
return (system_clock::now() < __atime
? cv_status::no_timeout : cv_status::timeout);
}
};
void
notify_all_at_thread_exit(condition_variable&, unique_lock<mutex>);
struct __at_thread_exit_elt
{
__at_thread_exit_elt* _M_next;
void (*_M_cb)(void*);
};
inline namespace _V2 {
class condition_variable_any
{
using __clock_t = chrono::steady_clock;
condition_variable _M_cond;
shared_ptr<mutex> _M_mutex;
template<typename _Lock>
struct _Unlock
{
explicit _Unlock(_Lock& __lk) : _M_lock(__lk) { __lk.unlock(); }
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
~_Unlock() noexcept(false)
{
if (uncaught_exception())
{
try
{ _M_lock.lock(); }
catch(const __cxxabiv1::__forced_unwind&)
{ throw; }
catch(...)
{ }
}
else
_M_lock.lock();
}
#pragma GCC diagnostic pop
_Unlock(const _Unlock&) = delete;
_Unlock& operator=(const _Unlock&) = delete;
_Lock& _M_lock;
};
public:
condition_variable_any() : _M_mutex(std::make_shared<mutex>()) { }
~condition_variable_any() = default;
condition_variable_any(const condition_variable_any&) = delete;
condition_variable_any& operator=(const condition_variable_any&) = delete;
void
notify_one() noexcept
{
lock_guard<mutex> __lock(*_M_mutex);
_M_cond.notify_one();
}
void
notify_all() noexcept
{
lock_guard<mutex> __lock(*_M_mutex);
_M_cond.notify_all();
}
template<typename _Lock>
void
wait(_Lock& __lock)
{
shared_ptr<mutex> __mutex = _M_mutex;
unique_lock<mutex> __my_lock(*__mutex);
_Unlock<_Lock> __unlock(__lock);
unique_lock<mutex> __my_lock2(std::move(__my_lock));
_M_cond.wait(__my_lock2);
}
template<typename _Lock, typename _Predicate>
void
wait(_Lock& __lock, _Predicate __p)
{
while (!__p())
wait(__lock);
}
template<typename _Lock, typename _Clock, typename _Duration>
cv_status
wait_until(_Lock& __lock,
const chrono::time_point<_Clock, _Duration>& __atime)
{
shared_ptr<mutex> __mutex = _M_mutex;
unique_lock<mutex> __my_lock(*__mutex);
_Unlock<_Lock> __unlock(__lock);
unique_lock<mutex> __my_lock2(std::move(__my_lock));
return _M_cond.wait_until(__my_lock2, __atime);
}
template<typename _Lock, typename _Clock,
typename _Duration, typename _Predicate>
bool
wait_until(_Lock& __lock,
const chrono::time_point<_Clock, _Duration>& __atime,
_Predicate __p)
{
while (!__p())
if (wait_until(__lock, __atime) == cv_status::timeout)
return __p();
return true;
}
template<typename _Lock, typename _Rep, typename _Period>
cv_status
wait_for(_Lock& __lock, const chrono::duration<_Rep, _Period>& __rtime)
{ return wait_until(__lock, __clock_t::now() + __rtime); }
template<typename _Lock, typename _Rep,
typename _Period, typename _Predicate>
bool
wait_for(_Lock& __lock,
const chrono::duration<_Rep, _Period>& __rtime, _Predicate __p)
{ return wait_until(__lock, __clock_t::now() + __rtime, std::move(__p)); }
};
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
class __recursive_mutex_base
{
protected:
typedef __gthread_recursive_mutex_t __native_type;
__recursive_mutex_base(const __recursive_mutex_base&) = delete;
__recursive_mutex_base& operator=(const __recursive_mutex_base&) = delete;
__native_type _M_mutex = { { 0, 0, 0, PTHREAD_MUTEX_RECURSIVE_NP, 0, { { 0, 0 } } } };
__recursive_mutex_base() = default;
};
class recursive_mutex : private __recursive_mutex_base
{
public:
typedef __native_type* native_handle_type;
recursive_mutex() = default;
~recursive_mutex() = default;
recursive_mutex(const recursive_mutex&) = delete;
recursive_mutex& operator=(const recursive_mutex&) = delete;
void
lock()
{
int __e = __gthread_recursive_mutex_lock(&_M_mutex);
if (__e)
__throw_system_error(__e);
}
[[__nodiscard__]]
bool
try_lock() noexcept
{
return !__gthread_recursive_mutex_trylock(&_M_mutex);
}
void
unlock()
{
__gthread_recursive_mutex_unlock(&_M_mutex);
}
native_handle_type
native_handle() noexcept
{ return &_M_mutex; }
};
template<typename _Derived>
class __timed_mutex_impl
{
protected:
template<typename _Rep, typename _Period>
bool
_M_try_lock_for(const chrono::duration<_Rep, _Period>& __rtime)
{
using __clock = chrono::steady_clock;
auto __rt = chrono::duration_cast<__clock::duration>(__rtime);
if (ratio_greater<__clock::period, _Period>())
++__rt;
return _M_try_lock_until(__clock::now() + __rt);
}
template<typename _Duration>
bool
_M_try_lock_until(const chrono::time_point<chrono::system_clock,
_Duration>& __atime)
{
auto __s = chrono::time_point_cast<chrono::seconds>(__atime);
auto __ns = chrono::duration_cast<chrono::nanoseconds>(__atime - __s);
__gthread_time_t __ts = {
static_cast<std::time_t>(__s.time_since_epoch().count()),
static_cast<long>(__ns.count())
};
return static_cast<_Derived*>(this)->_M_timedlock(__ts);
}
template<typename _Duration>
bool
_M_try_lock_until(const chrono::time_point<chrono::steady_clock,
_Duration>& __atime)
{
auto __s = chrono::time_point_cast<chrono::seconds>(__atime);
auto __ns = chrono::duration_cast<chrono::nanoseconds>(__atime - __s);
__gthread_time_t __ts = {
static_cast<std::time_t>(__s.time_since_epoch().count()),
static_cast<long>(__ns.count())
};
return static_cast<_Derived*>(this)->_M_clocklock(1,
__ts);
}
template<typename _Clock, typename _Duration>
bool
_M_try_lock_until(const chrono::time_point<_Clock, _Duration>& __atime)
{
auto __now = _Clock::now();
do {
auto __rtime = __atime - __now;
if (_M_try_lock_for(__rtime))
return true;
__now = _Clock::now();
} while (__atime > __now);
return false;
}
};
class timed_mutex
: private __mutex_base, public __timed_mutex_impl<timed_mutex>
{
public:
typedef __native_type* native_handle_type;
timed_mutex() = default;
~timed_mutex() = default;
timed_mutex(const timed_mutex&) = delete;
timed_mutex& operator=(const timed_mutex&) = delete;
void
lock()
{
int __e = __gthread_mutex_lock(&_M_mutex);
if (__e)
__throw_system_error(__e);
}
[[__nodiscard__]]
bool
try_lock() noexcept
{
return !__gthread_mutex_trylock(&_M_mutex);
}
template <class _Rep, class _Period>
[[__nodiscard__]]
bool
try_lock_for(const chrono::duration<_Rep, _Period>& __rtime)
{ return _M_try_lock_for(__rtime); }
template <class _Clock, class _Duration>
[[__nodiscard__]]
bool
try_lock_until(const chrono::time_point<_Clock, _Duration>& __atime)
{ return _M_try_lock_until(__atime); }
void
unlock()
{
__gthread_mutex_unlock(&_M_mutex);
}
native_handle_type
native_handle() noexcept
{ return &_M_mutex; }
private:
friend class __timed_mutex_impl<timed_mutex>;
bool
_M_timedlock(const __gthread_time_t& __ts)
{ return !__gthread_mutex_timedlock(&_M_mutex, &__ts); }
bool
_M_clocklock(clockid_t __clockid, const __gthread_time_t& __ts)
{ return !pthread_mutex_clocklock(&_M_mutex, __clockid, &__ts); }
};
class recursive_timed_mutex
: private __recursive_mutex_base,
public __timed_mutex_impl<recursive_timed_mutex>
{
public:
typedef __native_type* native_handle_type;
recursive_timed_mutex() = default;
~recursive_timed_mutex() = default;
recursive_timed_mutex(const recursive_timed_mutex&) = delete;
recursive_timed_mutex& operator=(const recursive_timed_mutex&) = delete;
void
lock()
{
int __e = __gthread_recursive_mutex_lock(&_M_mutex);
if (__e)
__throw_system_error(__e);
}
[[__nodiscard__]]
bool
try_lock() noexcept
{
return !__gthread_recursive_mutex_trylock(&_M_mutex);
}
template <class _Rep, class _Period>
[[__nodiscard__]]
bool
try_lock_for(const chrono::duration<_Rep, _Period>& __rtime)
{ return _M_try_lock_for(__rtime); }
template <class _Clock, class _Duration>
[[__nodiscard__]]
bool
try_lock_until(const chrono::time_point<_Clock, _Duration>& __atime)
{ return _M_try_lock_until(__atime); }
void
unlock()
{
__gthread_recursive_mutex_unlock(&_M_mutex);
}
native_handle_type
native_handle() noexcept
{ return &_M_mutex; }
private:
friend class __timed_mutex_impl<recursive_timed_mutex>;
bool
_M_timedlock(const __gthread_time_t& __ts)
{ return !__gthread_recursive_mutex_timedlock(&_M_mutex, &__ts); }
bool
_M_clocklock(clockid_t __clockid, const __gthread_time_t& __ts)
{ return !pthread_mutex_clocklock(&_M_mutex, __clockid, &__ts); }
};
namespace __detail
{
template<typename _Lockable>
inline int
__try_lock_impl(_Lockable& __l)
{
if (unique_lock<_Lockable> __lock{__l, try_to_lock})
{
__lock.release();
return -1;
}
else
return 0;
}
template<typename _L0, typename... _Lockables>
inline int
__try_lock_impl(_L0& __l0, _Lockables&... __lockables)
{
if constexpr ((is_same_v<_L0, _Lockables> && ...))
{
constexpr int _Np = 1 + sizeof...(_Lockables);
unique_lock<_L0> __locks[_Np] = {
{__l0, defer_lock}, {__lockables, defer_lock}...
};
for (int __i = 0; __i < _Np; ++__i)
{
if (!__locks[__i].try_lock())
{
const int __failed = __i;
while (__i--)
__locks[__i].unlock();
return __failed;
}
}
for (auto& __l : __locks)
__l.release();
return -1;
}
else
if (unique_lock<_L0> __lock{__l0, try_to_lock})
{
int __idx = __detail::__try_lock_impl(__lockables...);
if (__idx == -1)
{
__lock.release();
return -1;
}
return __idx + 1;
}
else
return 0;
}
}
template<typename _L1, typename _L2, typename... _L3>
[[__nodiscard__]]
inline int
try_lock(_L1& __l1, _L2& __l2, _L3&... __l3)
{
return __detail::__try_lock_impl(__l1, __l2, __l3...);
}
namespace __detail
{
template<typename _L0, typename... _L1>
void
__lock_impl(int& __i, int __depth, _L0& __l0, _L1&... __l1)
{
while (__i >= __depth)
{
if (__i == __depth)
{
int __failed = 1;
{
unique_lock<_L0> __first(__l0);
__failed += __detail::__try_lock_impl(__l1...);
if (!__failed)
{
__i = -1;
__first.release();
return;
}
}
__gthread_yield();
constexpr auto __n = 1 + sizeof...(_L1);
__i = (__depth + __failed) % __n;
}
else
__detail::__lock_impl(__i, __depth + 1, __l1..., __l0);
}
}
}
template<typename _L1, typename _L2, typename... _L3>
void
lock(_L1& __l1, _L2& __l2, _L3&... __l3)
{
if constexpr (is_same_v<_L1, _L2> && (is_same_v<_L1, _L3> && ...))
{
constexpr int _Np = 2 + sizeof...(_L3);
unique_lock<_L1> __locks[] = {
{__l1, defer_lock}, {__l2, defer_lock}, {__l3, defer_lock}...
};
int __first = 0;
do {
__locks[__first].lock();
for (int __j = 1; __j < _Np; ++__j)
{
const int __idx = (__first + __j) % _Np;
if (!__locks[__idx].try_lock())
{
for (int __k = __j; __k != 0; --__k)
__locks[(__first + __k - 1) % _Np].unlock();
__first = __idx;
break;
}
}
} while (!__locks[__first].owns_lock());
for (auto& __l : __locks)
__l.release();
}
else
{
int __i = 0;
__detail::__lock_impl(__i, 0, __l1, __l2, __l3...);
}
}
template<typename... _MutexTypes>
class scoped_lock
{
public:
explicit scoped_lock(_MutexTypes&... __m) : _M_devices(std::tie(__m...))
{ std::lock(__m...); }
explicit scoped_lock(adopt_lock_t, _MutexTypes&... __m) noexcept
: _M_devices(std::tie(__m...))
{ }
~scoped_lock()
{ std::apply([](auto&... __m) { (__m.unlock(), ...); }, _M_devices); }
scoped_lock(const scoped_lock&) = delete;
scoped_lock& operator=(const scoped_lock&) = delete;
private:
tuple<_MutexTypes&...> _M_devices;
};
template<>
class scoped_lock<>
{
public:
explicit scoped_lock() = default;
explicit scoped_lock(adopt_lock_t) noexcept { }
~scoped_lock() = default;
scoped_lock(const scoped_lock&) = delete;
scoped_lock& operator=(const scoped_lock&) = delete;
};
template<typename _Mutex>
class scoped_lock<_Mutex>
{
public:
using mutex_type = _Mutex;
explicit scoped_lock(mutex_type& __m) : _M_device(__m)
{ _M_device.lock(); }
explicit scoped_lock(adopt_lock_t, mutex_type& __m) noexcept
: _M_device(__m)
{ }
~scoped_lock()
{ _M_device.unlock(); }
scoped_lock(const scoped_lock&) = delete;
scoped_lock& operator=(const scoped_lock&) = delete;
private:
mutex_type& _M_device;
};
struct once_flag
{
constexpr once_flag() noexcept = default;
once_flag(const once_flag&) = delete;
once_flag& operator=(const once_flag&) = delete;
private:
__gthread_once_t _M_once = 0;
struct _Prepare_execution;
template<typename _Callable, typename... _Args>
friend void
call_once(once_flag& __once, _Callable&& __f, _Args&&... __args);
};
extern __thread void* __once_callable;
extern __thread void (*__once_call)();
struct once_flag::_Prepare_execution
{
template<typename _Callable>
explicit
_Prepare_execution(_Callable& __c)
{
__once_callable = std::__addressof(__c);
__once_call = [] { (*static_cast<_Callable*>(__once_callable))(); };
}
~_Prepare_execution()
{
__once_callable = nullptr;
__once_call = nullptr;
}
_Prepare_execution(const _Prepare_execution&) = delete;
_Prepare_execution& operator=(const _Prepare_execution&) = delete;
};
extern "C" void __once_proxy(void);
template<typename _Callable, typename... _Args>
void
call_once(once_flag& __once, _Callable&& __f, _Args&&... __args)
{
auto __callable = [&] {
std::__invoke(std::forward<_Callable>(__f),
std::forward<_Args>(__args)...);
};
once_flag::_Prepare_execution __exec(__callable);
if (int __e = __gthread_once(&__once._M_once, &__once_proxy))
__throw_system_error(__e);
}
}
namespace testing {
class Message;
using std::get;
using std::make_tuple;
using std::tuple;
using std::tuple_element;
using std::tuple_size;
namespace internal {
class Secret;
__attribute__((visibility("default"))) bool IsTrue(bool condition);
class __attribute__((visibility("default"))) RE {
public:
RE(const RE& other) { Init(other.pattern()); }
RE(const ::std::string& regex) { Init(regex.c_str()); }
RE(const char* regex) { Init(regex); }
~RE();
const char* pattern() const { return pattern_; }
static bool FullMatch(const ::std::string& str, const RE& re) {
return FullMatch(str.c_str(), re);
}
static bool PartialMatch(const ::std::string& str, const RE& re) {
return PartialMatch(str.c_str(), re);
}
static bool FullMatch(const char* str, const RE& re);
static bool PartialMatch(const char* str, const RE& re);
private:
void Init(const char* regex);
const char* pattern_;
bool is_valid_;
regex_t full_regex_;
regex_t partial_regex_;
};
__attribute__((visibility("default"))) ::std::string FormatFileLocation(const char* file, int line);
__attribute__((visibility("default"))) ::std::string FormatCompilerIndependentFileLocation(const char* file,
int line);
enum GTestLogSeverity { GTEST_INFO, GTEST_WARNING, GTEST_ERROR, GTEST_FATAL };
class __attribute__((visibility("default"))) GTestLog {
public:
GTestLog(GTestLogSeverity severity, const char* file, int line);
~GTestLog();
::std::ostream& GetStream() { return ::std::cerr; }
private:
const GTestLogSeverity severity_;
GTestLog(const GTestLog&) = delete;
GTestLog& operator=(const GTestLog&) = delete;
};
inline void LogToStderr() {}
inline void FlushInfoLog() { fflush(nullptr); }
template <typename T>
struct ConstRef {
typedef const T& type;
};
template <typename T>
struct ConstRef<T&> {
typedef T& type;
};
template <typename To>
inline To ImplicitCast_(To x) {
return x;
}
template <typename To, typename From>
inline To DownCast_(From* f) {
if (false) {
const To to = nullptr;
::testing::internal::ImplicitCast_<From*>(to);
}
switch (0) case 0: default: if (::testing::internal::IsTrue(f == nullptr || dynamic_cast<To>(f) != nullptr)) ; else ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-port.h", 1127) .GetStream() << "Condition " "f == nullptr || dynamic_cast<To>(f) != nullptr" " failed. ";
return static_cast<To>(f);
}
template <class Derived, class Base>
Derived* CheckedDowncastToActualType(Base* base) {
switch (0) case 0: default: if (::testing::internal::IsTrue(typeid(*base) == typeid(Derived))) ; else ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-port.h", 1140) .GetStream() << "Condition " "typeid(*base) == typeid(Derived)" " failed. ";
return dynamic_cast<Derived*>(base);
}
__attribute__((visibility("default"))) void CaptureStdout();
__attribute__((visibility("default"))) std::string GetCapturedStdout();
__attribute__((visibility("default"))) void CaptureStderr();
__attribute__((visibility("default"))) std::string GetCapturedStderr();
__attribute__((visibility("default"))) size_t GetFileSize(FILE* file);
__attribute__((visibility("default"))) std::string ReadEntireFile(FILE* file);
__attribute__((visibility("default"))) std::vector<std::string> GetArgvs();
std::vector<std::string> GetInjectableArgvs();
void SetInjectableArgvs(const std::vector<std::string>* new_argvs);
void SetInjectableArgvs(const std::vector<std::string>& new_argvs);
void ClearInjectableArgvs();
class __attribute__((visibility("default"))) Notification {
public:
Notification() : notified_(false) {}
Notification(const Notification&) = delete;
Notification& operator=(const Notification&) = delete;
void Notify() {
std::lock_guard<std::mutex> lock(mu_);
notified_ = true;
cv_.notify_all();
}
void WaitForNotification() {
std::unique_lock<std::mutex> lock(mu_);
cv_.wait(lock, [this]() { return notified_; });
}
private:
std::mutex mu_;
std::condition_variable cv_;
bool notified_;
};
class ThreadWithParamBase {
public:
virtual ~ThreadWithParamBase() {}
virtual void Run() = 0;
};
extern "C" inline void* ThreadFuncWithCLinkage(void* thread) {
static_cast<ThreadWithParamBase*>(thread)->Run();
return nullptr;
}
template <typename T>
class ThreadWithParam : public ThreadWithParamBase {
public:
typedef void UserThreadFunc(T);
ThreadWithParam(UserThreadFunc* func, T param, Notification* thread_can_start)
: func_(func),
param_(param),
thread_can_start_(thread_can_start),
finished_(false) {
ThreadWithParamBase* const base = this;
if (const int gtest_error = (pthread_create(&thread_, nullptr, &ThreadFuncWithCLinkage, base))) ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-port.h", 1316) .GetStream() << "pthread_create(&thread_, nullptr, &ThreadFuncWithCLinkage, base)" << "failed with error " << gtest_error
;
}
~ThreadWithParam() override { Join(); }
void Join() {
if (!finished_) {
if (const int gtest_error = (pthread_join(thread_, nullptr))) ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-port.h", 1323) .GetStream() << "pthread_join(thread_, nullptr)" << "failed with error " << gtest_error;
finished_ = true;
}
}
void Run() override {
if (thread_can_start_ != nullptr) thread_can_start_->WaitForNotification();
func_(param_);
}
private:
UserThreadFunc* const func_;
const T param_;
Notification* const thread_can_start_;
bool finished_;
pthread_t thread_;
ThreadWithParam(const ThreadWithParam&) = delete;
ThreadWithParam& operator=(const ThreadWithParam&) = delete;
};
class MutexBase {
public:
void Lock() {
if (const int gtest_error = (pthread_mutex_lock(&mutex_))) ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-port.h", 1642) .GetStream() << "pthread_mutex_lock(&mutex_)" << "failed with error " << gtest_error;
owner_ = pthread_self();
has_owner_ = true;
}
void Unlock() {
has_owner_ = false;
if (const int gtest_error = (pthread_mutex_unlock(&mutex_))) ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-port.h", 1654) .GetStream() << "pthread_mutex_unlock(&mutex_)" << "failed with error " << gtest_error;
}
void AssertHeld() const {
switch (0) case 0: default: if (::testing::internal::IsTrue(has_owner_ && pthread_equal(owner_, pthread_self()))) ; else ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-port.h", 1660) .GetStream() << "Condition " "has_owner_ && pthread_equal(owner_, pthread_self())" " failed. "
<< "The current thread is not holding the mutex @" << this;
}
public:
pthread_mutex_t mutex_;
bool has_owner_;
pthread_t owner_;
};
class Mutex : public MutexBase {
public:
Mutex() {
if (const int gtest_error = (pthread_mutex_init(&mutex_, nullptr))) ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-port.h", 1699) .GetStream() << "pthread_mutex_init(&mutex_, nullptr)" << "failed with error " << gtest_error;
has_owner_ = false;
}
~Mutex() { if (const int gtest_error = (pthread_mutex_destroy(&mutex_))) ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-port.h", 1702) .GetStream() << "pthread_mutex_destroy(&mutex_)" << "failed with error " << gtest_error; }
private:
Mutex(const Mutex&) = delete;
Mutex& operator=(const Mutex&) = delete;
};
class GTestMutexLock {
public:
explicit GTestMutexLock(MutexBase* mutex) : mutex_(mutex) { mutex_->Lock(); }
~GTestMutexLock() { mutex_->Unlock(); }
private:
MutexBase* const mutex_;
GTestMutexLock(const GTestMutexLock&) = delete;
GTestMutexLock& operator=(const GTestMutexLock&) = delete;
};
typedef GTestMutexLock MutexLock;
class __attribute__((visibility("default"))) ThreadLocalValueHolderBase {
public:
virtual ~ThreadLocalValueHolderBase() {}
};
extern "C" inline void DeleteThreadLocalValue(void* value_holder) {
delete static_cast<ThreadLocalValueHolderBase*>(value_holder);
}
template <typename T>
class __attribute__((visibility("default"))) ThreadLocal {
public:
ThreadLocal()
: key_(CreateKey()), default_factory_(new DefaultValueHolderFactory()) {}
explicit ThreadLocal(const T& value)
: key_(CreateKey()),
default_factory_(new InstanceValueHolderFactory(value)) {}
~ThreadLocal() {
DeleteThreadLocalValue(pthread_getspecific(key_));
if (const int gtest_error = (pthread_key_delete(key_))) ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-port.h", 1762) .GetStream() << "pthread_key_delete(key_)" << "failed with error " << gtest_error;
}
T* pointer() { return GetOrCreateValue(); }
const T* pointer() const { return GetOrCreateValue(); }
const T& get() const { return *pointer(); }
void set(const T& value) { *pointer() = value; }
private:
class ValueHolder : public ThreadLocalValueHolderBase {
public:
ValueHolder() : value_() {}
explicit ValueHolder(const T& value) : value_(value) {}
T* pointer() { return &value_; }
private:
T value_;
ValueHolder(const ValueHolder&) = delete;
ValueHolder& operator=(const ValueHolder&) = delete;
};
static pthread_key_t CreateKey() {
pthread_key_t key;
if (const int gtest_error = (pthread_key_create(&key, &DeleteThreadLocalValue))) ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-port.h", 1789) .GetStream() << "pthread_key_create(&key, &DeleteThreadLocalValue)" << "failed with error " << gtest_error
;
return key;
}
T* GetOrCreateValue() const {
ThreadLocalValueHolderBase* const holder =
static_cast<ThreadLocalValueHolderBase*>(pthread_getspecific(key_));
if (holder != nullptr) {
return CheckedDowncastToActualType<ValueHolder>(holder)->pointer();
}
ValueHolder* const new_holder = default_factory_->MakeNewHolder();
ThreadLocalValueHolderBase* const holder_base = new_holder;
if (const int gtest_error = (pthread_setspecific(key_, holder_base))) ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-port.h", 1803) .GetStream() << "pthread_setspecific(key_, holder_base)" << "failed with error " << gtest_error;
return new_holder->pointer();
}
class ValueHolderFactory {
public:
ValueHolderFactory() {}
virtual ~ValueHolderFactory() {}
virtual ValueHolder* MakeNewHolder() const = 0;
private:
ValueHolderFactory(const ValueHolderFactory&) = delete;
ValueHolderFactory& operator=(const ValueHolderFactory&) = delete;
};
class DefaultValueHolderFactory : public ValueHolderFactory {
public:
DefaultValueHolderFactory() {}
ValueHolder* MakeNewHolder() const override { return new ValueHolder(); }
private:
DefaultValueHolderFactory(const DefaultValueHolderFactory&) = delete;
DefaultValueHolderFactory& operator=(const DefaultValueHolderFactory&) =
delete;
};
class InstanceValueHolderFactory : public ValueHolderFactory {
public:
explicit InstanceValueHolderFactory(const T& value) : value_(value) {}
ValueHolder* MakeNewHolder() const override {
return new ValueHolder(value_);
}
private:
const T value_;
InstanceValueHolderFactory(const InstanceValueHolderFactory&) = delete;
InstanceValueHolderFactory& operator=(const InstanceValueHolderFactory&) =
delete;
};
const pthread_key_t key_;
std::unique_ptr<ValueHolderFactory> default_factory_;
ThreadLocal(const ThreadLocal&) = delete;
ThreadLocal& operator=(const ThreadLocal&) = delete;
};
__attribute__((visibility("default"))) size_t GetThreadCount();
inline bool IsAlpha(char ch) {
return isalpha(static_cast<unsigned char>(ch)) != 0;
}
inline bool IsAlNum(char ch) {
return isalnum(static_cast<unsigned char>(ch)) != 0;
}
inline bool IsDigit(char ch) {
return isdigit(static_cast<unsigned char>(ch)) != 0;
}
inline bool IsLower(char ch) {
return islower(static_cast<unsigned char>(ch)) != 0;
}
inline bool IsSpace(char ch) {
return isspace(static_cast<unsigned char>(ch)) != 0;
}
inline bool IsUpper(char ch) {
return isupper(static_cast<unsigned char>(ch)) != 0;
}
inline bool IsXDigit(char ch) {
return isxdigit(static_cast<unsigned char>(ch)) != 0;
}
inline bool IsXDigit(char16_t ch) {
const unsigned char low_byte = static_cast<unsigned char>(ch);
return ch == low_byte && isxdigit(low_byte) != 0;
}
inline bool IsXDigit(char32_t ch) {
const unsigned char low_byte = static_cast<unsigned char>(ch);
return ch == low_byte && isxdigit(low_byte) != 0;
}
inline bool IsXDigit(wchar_t ch) {
const unsigned char low_byte = static_cast<unsigned char>(ch);
return ch == low_byte && isxdigit(low_byte) != 0;
}
inline char ToLower(char ch) {
return static_cast<char>(tolower(static_cast<unsigned char>(ch)));
}
inline char ToUpper(char ch) {
return static_cast<char>(toupper(static_cast<unsigned char>(ch)));
}
inline std::string StripTrailingSpaces(std::string str) {
std::string::iterator it = str.end();
while (it != str.begin() && IsSpace(*--it)) it = str.erase(it);
return str;
}
namespace posix {
typedef struct stat StatStruct;
inline int FileNo(FILE* file) { return fileno(file); }
inline int Stat(const char* path, StatStruct* buf) { return stat(path, buf); }
inline int RmDir(const char* dir) { return rmdir(dir); }
inline bool IsDir(const StatStruct& st) { return ((((st.st_mode)) & 0170000) == (0040000)); }
inline int DoIsATTY(int fd) { return isatty(fd); }
inline int StrCaseCmp(const char* s1, const char* s2) {
return strcasecmp(s1, s2);
}
inline char* StrDup(const char* src) { return strdup(src); }
inline int IsATTY(int fd) {
int savedErrno = (*__errno_location ());
int isAttyValue = DoIsATTY(fd);
(*__errno_location ()) = savedErrno;
return isAttyValue;
}
inline int ChDir(const char* dir) { return chdir(dir); }
inline FILE* FOpen(const char* path, const char* mode) {
return fopen(path, mode);
}
inline FILE* FReopen(const char* path, const char* mode, FILE* stream) {
return freopen(path, mode, stream);
}
inline FILE* FDOpen(int fd, const char* mode) { return fdopen(fd, mode); }
inline int FClose(FILE* fp) { return fclose(fp); }
inline int Read(int fd, void* buf, unsigned int count) {
return static_cast<int>(read(fd, buf, count));
}
inline int Write(int fd, const void* buf, unsigned int count) {
return static_cast<int>(write(fd, buf, count));
}
inline int Close(int fd) { return close(fd); }
inline const char* StrError(int errnum) { return strerror(errnum); }
inline const char* GetEnv(const char* name) {
return getenv(name);
}
[[noreturn]] inline void Abort() { abort(); }
}
using BiggestInt = long long;
constexpr BiggestInt kMaxBiggestInt = (std::numeric_limits<BiggestInt>::max)();
template <size_t size>
class TypeWithSize {
public:
using UInt = void;
};
template <>
class TypeWithSize<4> {
public:
using Int = std::int32_t;
using UInt = std::uint32_t;
};
template <>
class TypeWithSize<8> {
public:
using Int = std::int64_t;
using UInt = std::uint64_t;
};
using TimeInMillis = int64_t;
__attribute__((visibility("default"))) bool ParseInt32(const Message& src_text, const char* str,
int32_t* value);
bool BoolFromGTestEnv(const char* flag, bool default_val);
__attribute__((visibility("default"))) int32_t Int32FromGTestEnv(const char* flag, int32_t default_val);
std::string OutputFlagAlsoCheckEnvVar();
const char* StringFromGTestEnv(const char* flag, const char* default_val);
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
class bad_any_cast : public bad_cast
{
public:
virtual const char* what() const noexcept { return "bad any_cast"; }
};
[[gnu::noreturn]] inline void __throw_bad_any_cast()
{
throw bad_any_cast{};
}
class any
{
union _Storage
{
constexpr _Storage() : _M_ptr{nullptr} {}
_Storage(const _Storage&) = delete;
_Storage& operator=(const _Storage&) = delete;
void* _M_ptr;
aligned_storage<sizeof(_M_ptr), alignof(void*)>::type _M_buffer;
};
template<typename _Tp, typename _Safe = is_nothrow_move_constructible<_Tp>,
bool _Fits = (sizeof(_Tp) <= sizeof(_Storage))
&& (alignof(_Tp) <= alignof(_Storage))>
using _Internal = std::integral_constant<bool, _Safe::value && _Fits>;
template<typename _Tp>
struct _Manager_internal;
template<typename _Tp>
struct _Manager_external;
template<typename _Tp>
using _Manager = __conditional_t<_Internal<_Tp>::value,
_Manager_internal<_Tp>,
_Manager_external<_Tp>>;
template<typename _Tp, typename _VTp = decay_t<_Tp>>
using _Decay_if_not_any = enable_if_t<!is_same_v<_VTp, any>, _VTp>;
template <typename _Tp, typename... _Args,
typename _Mgr = _Manager<_Tp>>
void __do_emplace(_Args&&... __args)
{
reset();
_Mgr::_S_create(_M_storage, std::forward<_Args>(__args)...);
_M_manager = &_Mgr::_S_manage;
}
template <typename _Tp, typename _Up, typename... _Args,
typename _Mgr = _Manager<_Tp>>
void __do_emplace(initializer_list<_Up> __il, _Args&&... __args)
{
reset();
_Mgr::_S_create(_M_storage, __il, std::forward<_Args>(__args)...);
_M_manager = &_Mgr::_S_manage;
}
template <typename _Res, typename _Tp, typename... _Args>
using __any_constructible
= enable_if<__and_<is_copy_constructible<_Tp>,
is_constructible<_Tp, _Args...>>::value,
_Res>;
template <typename _Tp, typename... _Args>
using __any_constructible_t
= typename __any_constructible<bool, _Tp, _Args...>::type;
template<typename _VTp, typename... _Args>
using __emplace_t
= typename __any_constructible<_VTp&, _VTp, _Args...>::type;
public:
constexpr any() noexcept : _M_manager(nullptr) { }
any(const any& __other)
{
if (!__other.has_value())
_M_manager = nullptr;
else
{
_Arg __arg;
__arg._M_any = this;
__other._M_manager(_Op_clone, &__other, &__arg);
}
}
any(any&& __other) noexcept
{
if (!__other.has_value())
_M_manager = nullptr;
else
{
_Arg __arg;
__arg._M_any = this;
__other._M_manager(_Op_xfer, &__other, &__arg);
}
}
template <typename _Tp, typename _VTp = _Decay_if_not_any<_Tp>,
typename _Mgr = _Manager<_VTp>,
enable_if_t<is_copy_constructible_v<_VTp>
&& !__is_in_place_type_v<_VTp>, bool> = true>
any(_Tp&& __value)
: _M_manager(&_Mgr::_S_manage)
{
_Mgr::_S_create(_M_storage, std::forward<_Tp>(__value));
}
template <typename _Tp, typename... _Args, typename _VTp = decay_t<_Tp>,
typename _Mgr = _Manager<_VTp>,
__any_constructible_t<_VTp, _Args&&...> = false>
explicit
any(in_place_type_t<_Tp>, _Args&&... __args)
: _M_manager(&_Mgr::_S_manage)
{
_Mgr::_S_create(_M_storage, std::forward<_Args>(__args)...);
}
template <typename _Tp, typename _Up, typename... _Args,
typename _VTp = decay_t<_Tp>, typename _Mgr = _Manager<_VTp>,
__any_constructible_t<_VTp, initializer_list<_Up>&,
_Args&&...> = false>
explicit
any(in_place_type_t<_Tp>, initializer_list<_Up> __il, _Args&&... __args)
: _M_manager(&_Mgr::_S_manage)
{
_Mgr::_S_create(_M_storage, __il, std::forward<_Args>(__args)...);
}
~any() { reset(); }
any&
operator=(const any& __rhs)
{
*this = any(__rhs);
return *this;
}
any&
operator=(any&& __rhs) noexcept
{
if (!__rhs.has_value())
reset();
else if (this != &__rhs)
{
reset();
_Arg __arg;
__arg._M_any = this;
__rhs._M_manager(_Op_xfer, &__rhs, &__arg);
}
return *this;
}
template<typename _Tp>
enable_if_t<is_copy_constructible<_Decay_if_not_any<_Tp>>::value, any&>
operator=(_Tp&& __rhs)
{
*this = any(std::forward<_Tp>(__rhs));
return *this;
}
template <typename _Tp, typename... _Args>
__emplace_t<decay_t<_Tp>, _Args...>
emplace(_Args&&... __args)
{
using _VTp = decay_t<_Tp>;
__do_emplace<_VTp>(std::forward<_Args>(__args)...);
return *any::_Manager<_VTp>::_S_access(_M_storage);
}
template <typename _Tp, typename _Up, typename... _Args>
__emplace_t<decay_t<_Tp>, initializer_list<_Up>&, _Args&&...>
emplace(initializer_list<_Up> __il, _Args&&... __args)
{
using _VTp = decay_t<_Tp>;
__do_emplace<_VTp, _Up>(__il, std::forward<_Args>(__args)...);
return *any::_Manager<_VTp>::_S_access(_M_storage);
}
void reset() noexcept
{
if (has_value())
{
_M_manager(_Op_destroy, this, nullptr);
_M_manager = nullptr;
}
}
void swap(any& __rhs) noexcept
{
if (!has_value() && !__rhs.has_value())
return;
if (has_value() && __rhs.has_value())
{
if (this == &__rhs)
return;
any __tmp;
_Arg __arg;
__arg._M_any = &__tmp;
__rhs._M_manager(_Op_xfer, &__rhs, &__arg);
__arg._M_any = &__rhs;
_M_manager(_Op_xfer, this, &__arg);
__arg._M_any = this;
__tmp._M_manager(_Op_xfer, &__tmp, &__arg);
}
else
{
any* __empty = !has_value() ? this : &__rhs;
any* __full = !has_value() ? &__rhs : this;
_Arg __arg;
__arg._M_any = __empty;
__full->_M_manager(_Op_xfer, __full, &__arg);
}
}
bool has_value() const noexcept { return _M_manager != nullptr; }
const type_info& type() const noexcept
{
if (!has_value())
return typeid(void);
_Arg __arg;
_M_manager(_Op_get_type_info, this, &__arg);
return *__arg._M_typeinfo;
}
template<typename _Tp>
static constexpr bool __is_valid_cast()
{ return __or_<is_reference<_Tp>, is_copy_constructible<_Tp>>::value; }
private:
enum _Op {
_Op_access, _Op_get_type_info, _Op_clone, _Op_destroy, _Op_xfer
};
union _Arg
{
void* _M_obj;
const std::type_info* _M_typeinfo;
any* _M_any;
};
void (*_M_manager)(_Op, const any*, _Arg*);
_Storage _M_storage;
template<typename _Tp>
friend void* __any_caster(const any* __any);
template<typename _Tp>
struct _Manager_internal
{
static void
_S_manage(_Op __which, const any* __anyp, _Arg* __arg);
template<typename _Up>
static void
_S_create(_Storage& __storage, _Up&& __value)
{
void* __addr = &__storage._M_buffer;
::new (__addr) _Tp(std::forward<_Up>(__value));
}
template<typename... _Args>
static void
_S_create(_Storage& __storage, _Args&&... __args)
{
void* __addr = &__storage._M_buffer;
::new (__addr) _Tp(std::forward<_Args>(__args)...);
}
static _Tp*
_S_access(const _Storage& __storage)
{
const void* __addr = &__storage._M_buffer;
return static_cast<_Tp*>(const_cast<void*>(__addr));
}
};
template<typename _Tp>
struct _Manager_external
{
static void
_S_manage(_Op __which, const any* __anyp, _Arg* __arg);
template<typename _Up>
static void
_S_create(_Storage& __storage, _Up&& __value)
{
__storage._M_ptr = new _Tp(std::forward<_Up>(__value));
}
template<typename... _Args>
static void
_S_create(_Storage& __storage, _Args&&... __args)
{
__storage._M_ptr = new _Tp(std::forward<_Args>(__args)...);
}
static _Tp*
_S_access(const _Storage& __storage)
{
return static_cast<_Tp*>(__storage._M_ptr);
}
};
};
inline void swap(any& __x, any& __y) noexcept { __x.swap(__y); }
template <typename _Tp, typename... _Args>
inline
enable_if_t<is_constructible_v<any, in_place_type_t<_Tp>, _Args...>, any>
make_any(_Args&&... __args)
{
return any(in_place_type<_Tp>, std::forward<_Args>(__args)...);
}
template <typename _Tp, typename _Up, typename... _Args>
inline
enable_if_t<is_constructible_v<any, in_place_type_t<_Tp>,
initializer_list<_Up>&, _Args...>, any>
make_any(initializer_list<_Up> __il, _Args&&... __args)
{
return any(in_place_type<_Tp>, __il, std::forward<_Args>(__args)...);
}
template<typename _ValueType>
inline _ValueType any_cast(const any& __any)
{
using _Up = __remove_cvref_t<_ValueType>;
static_assert(any::__is_valid_cast<_ValueType>(),
"Template argument must be a reference or CopyConstructible type");
static_assert(is_constructible_v<_ValueType, const _Up&>,
"Template argument must be constructible from a const value.");
auto __p = any_cast<_Up>(&__any);
if (__p)
return static_cast<_ValueType>(*__p);
__throw_bad_any_cast();
}
template<typename _ValueType>
inline _ValueType any_cast(any& __any)
{
using _Up = __remove_cvref_t<_ValueType>;
static_assert(any::__is_valid_cast<_ValueType>(),
"Template argument must be a reference or CopyConstructible type");
static_assert(is_constructible_v<_ValueType, _Up&>,
"Template argument must be constructible from an lvalue.");
auto __p = any_cast<_Up>(&__any);
if (__p)
return static_cast<_ValueType>(*__p);
__throw_bad_any_cast();
}
template<typename _ValueType>
inline _ValueType any_cast(any&& __any)
{
using _Up = __remove_cvref_t<_ValueType>;
static_assert(any::__is_valid_cast<_ValueType>(),
"Template argument must be a reference or CopyConstructible type");
static_assert(is_constructible_v<_ValueType, _Up>,
"Template argument must be constructible from an rvalue.");
auto __p = any_cast<_Up>(&__any);
if (__p)
return static_cast<_ValueType>(std::move(*__p));
__throw_bad_any_cast();
}
template<typename _Tp>
void* __any_caster(const any* __any)
{
using _Up = remove_cv_t<_Tp>;
if constexpr (!is_same_v<decay_t<_Up>, _Up>)
return nullptr;
else if constexpr (!is_copy_constructible_v<_Up>)
return nullptr;
else if (__any->_M_manager == &any::_Manager<_Up>::_S_manage
|| __any->type() == typeid(_Tp)
)
{
return any::_Manager<_Up>::_S_access(__any->_M_storage);
}
return nullptr;
}
template<typename _ValueType>
inline const _ValueType* any_cast(const any* __any) noexcept
{
if constexpr (is_object_v<_ValueType>)
if (__any)
return static_cast<_ValueType*>(__any_caster<_ValueType>(__any));
return nullptr;
}
template<typename _ValueType>
inline _ValueType* any_cast(any* __any) noexcept
{
if constexpr (is_object_v<_ValueType>)
if (__any)
return static_cast<_ValueType*>(__any_caster<_ValueType>(__any));
return nullptr;
}
template<typename _Tp>
void
any::_Manager_internal<_Tp>::
_S_manage(_Op __which, const any* __any, _Arg* __arg)
{
auto __ptr = reinterpret_cast<const _Tp*>(&__any->_M_storage._M_buffer);
switch (__which)
{
case _Op_access:
__arg->_M_obj = const_cast<_Tp*>(__ptr);
break;
case _Op_get_type_info:
__arg->_M_typeinfo = &typeid(_Tp);
break;
case _Op_clone:
::new(&__arg->_M_any->_M_storage._M_buffer) _Tp(*__ptr);
__arg->_M_any->_M_manager = __any->_M_manager;
break;
case _Op_destroy:
__ptr->~_Tp();
break;
case _Op_xfer:
::new(&__arg->_M_any->_M_storage._M_buffer) _Tp
(std::move(*const_cast<_Tp*>(__ptr)));
__ptr->~_Tp();
__arg->_M_any->_M_manager = __any->_M_manager;
const_cast<any*>(__any)->_M_manager = nullptr;
break;
}
}
template<typename _Tp>
void
any::_Manager_external<_Tp>::
_S_manage(_Op __which, const any* __any, _Arg* __arg)
{
auto __ptr = static_cast<const _Tp*>(__any->_M_storage._M_ptr);
switch (__which)
{
case _Op_access:
__arg->_M_obj = const_cast<_Tp*>(__ptr);
break;
case _Op_get_type_info:
__arg->_M_typeinfo = &typeid(_Tp);
break;
case _Op_clone:
__arg->_M_any->_M_storage._M_ptr = new _Tp(*__ptr);
__arg->_M_any->_M_manager = __any->_M_manager;
break;
case _Op_destroy:
delete __ptr;
break;
case _Op_xfer:
__arg->_M_any->_M_storage._M_ptr = __any->_M_storage._M_ptr;
__arg->_M_any->_M_manager = __any->_M_manager;
const_cast<any*>(__any)->_M_manager = nullptr;
break;
}
}
namespace __detail::__variant
{
template<typename> struct _Never_valueless_alt;
template<>
struct _Never_valueless_alt<std::any>
: std::true_type
{ };
}
}
namespace testing {
namespace internal {
using Any = ::std::any;
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
struct _Enable_default_constructor_tag
{
explicit constexpr _Enable_default_constructor_tag() = default;
};
template<bool _Switch, typename _Tag = void>
struct _Enable_default_constructor
{
constexpr _Enable_default_constructor() noexcept = default;
constexpr _Enable_default_constructor(_Enable_default_constructor const&)
noexcept = default;
constexpr _Enable_default_constructor(_Enable_default_constructor&&)
noexcept = default;
_Enable_default_constructor&
operator=(_Enable_default_constructor const&) noexcept = default;
_Enable_default_constructor&
operator=(_Enable_default_constructor&&) noexcept = default;
constexpr explicit
_Enable_default_constructor(_Enable_default_constructor_tag) { }
};
template<bool _Switch, typename _Tag = void>
struct _Enable_destructor { };
template<bool _Copy, bool _CopyAssignment,
bool _Move, bool _MoveAssignment,
typename _Tag = void>
struct _Enable_copy_move { };
template<bool _Default, bool _Destructor,
bool _Copy, bool _CopyAssignment,
bool _Move, bool _MoveAssignment,
typename _Tag = void>
struct _Enable_special_members
: private _Enable_default_constructor<_Default, _Tag>,
private _Enable_destructor<_Destructor, _Tag>,
private _Enable_copy_move<_Copy, _CopyAssignment,
_Move, _MoveAssignment,
_Tag>
{ };
template<typename _Tag>
struct _Enable_default_constructor<false, _Tag>
{
constexpr _Enable_default_constructor() noexcept = delete;
constexpr _Enable_default_constructor(_Enable_default_constructor const&)
noexcept = default;
constexpr _Enable_default_constructor(_Enable_default_constructor&&)
noexcept = default;
_Enable_default_constructor&
operator=(_Enable_default_constructor const&) noexcept = default;
_Enable_default_constructor&
operator=(_Enable_default_constructor&&) noexcept = default;
constexpr explicit
_Enable_default_constructor(_Enable_default_constructor_tag) { }
};
template<typename _Tag>
struct _Enable_destructor<false, _Tag>
{ ~_Enable_destructor() noexcept = delete; };
template<typename _Tag>
struct _Enable_copy_move<false, true, true, true, _Tag>
{
constexpr _Enable_copy_move() noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move const&) noexcept = delete;
constexpr _Enable_copy_move(_Enable_copy_move&&) noexcept = default;
_Enable_copy_move&
operator=(_Enable_copy_move const&) noexcept = default;
_Enable_copy_move&
operator=(_Enable_copy_move&&) noexcept = default;
};
template<typename _Tag>
struct _Enable_copy_move<true, false, true, true, _Tag>
{
constexpr _Enable_copy_move() noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move const&) noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move&&) noexcept = default;
_Enable_copy_move&
operator=(_Enable_copy_move const&) noexcept = delete;
_Enable_copy_move&
operator=(_Enable_copy_move&&) noexcept = default;
};
template<typename _Tag>
struct _Enable_copy_move<false, false, true, true, _Tag>
{
constexpr _Enable_copy_move() noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move const&) noexcept = delete;
constexpr _Enable_copy_move(_Enable_copy_move&&) noexcept = default;
_Enable_copy_move&
operator=(_Enable_copy_move const&) noexcept = delete;
_Enable_copy_move&
operator=(_Enable_copy_move&&) noexcept = default;
};
template<typename _Tag>
struct _Enable_copy_move<true, true, false, true, _Tag>
{
constexpr _Enable_copy_move() noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move const&) noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move&&) noexcept = delete;
_Enable_copy_move&
operator=(_Enable_copy_move const&) noexcept = default;
_Enable_copy_move&
operator=(_Enable_copy_move&&) noexcept = default;
};
template<typename _Tag>
struct _Enable_copy_move<false, true, false, true, _Tag>
{
constexpr _Enable_copy_move() noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move const&) noexcept = delete;
constexpr _Enable_copy_move(_Enable_copy_move&&) noexcept = delete;
_Enable_copy_move&
operator=(_Enable_copy_move const&) noexcept = default;
_Enable_copy_move&
operator=(_Enable_copy_move&&) noexcept = default;
};
template<typename _Tag>
struct _Enable_copy_move<true, false, false, true, _Tag>
{
constexpr _Enable_copy_move() noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move const&) noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move&&) noexcept = delete;
_Enable_copy_move&
operator=(_Enable_copy_move const&) noexcept = delete;
_Enable_copy_move&
operator=(_Enable_copy_move&&) noexcept = default;
};
template<typename _Tag>
struct _Enable_copy_move<false, false, false, true, _Tag>
{
constexpr _Enable_copy_move() noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move const&) noexcept = delete;
constexpr _Enable_copy_move(_Enable_copy_move&&) noexcept = delete;
_Enable_copy_move&
operator=(_Enable_copy_move const&) noexcept = delete;
_Enable_copy_move&
operator=(_Enable_copy_move&&) noexcept = default;
};
template<typename _Tag>
struct _Enable_copy_move<true, true, true, false, _Tag>
{
constexpr _Enable_copy_move() noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move const&) noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move&&) noexcept = default;
_Enable_copy_move&
operator=(_Enable_copy_move const&) noexcept = default;
_Enable_copy_move&
operator=(_Enable_copy_move&&) noexcept = delete;
};
template<typename _Tag>
struct _Enable_copy_move<false, true, true, false, _Tag>
{
constexpr _Enable_copy_move() noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move const&) noexcept = delete;
constexpr _Enable_copy_move(_Enable_copy_move&&) noexcept = default;
_Enable_copy_move&
operator=(_Enable_copy_move const&) noexcept = default;
_Enable_copy_move&
operator=(_Enable_copy_move&&) noexcept = delete;
};
template<typename _Tag>
struct _Enable_copy_move<true, false, true, false, _Tag>
{
constexpr _Enable_copy_move() noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move const&) noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move&&) noexcept = default;
_Enable_copy_move&
operator=(_Enable_copy_move const&) noexcept = delete;
_Enable_copy_move&
operator=(_Enable_copy_move&&) noexcept = delete;
};
template<typename _Tag>
struct _Enable_copy_move<false, false, true, false, _Tag>
{
constexpr _Enable_copy_move() noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move const&) noexcept = delete;
constexpr _Enable_copy_move(_Enable_copy_move&&) noexcept = default;
_Enable_copy_move&
operator=(_Enable_copy_move const&) noexcept = delete;
_Enable_copy_move&
operator=(_Enable_copy_move&&) noexcept = delete;
};
template<typename _Tag>
struct _Enable_copy_move<true, true, false, false, _Tag>
{
constexpr _Enable_copy_move() noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move const&) noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move&&) noexcept = delete;
_Enable_copy_move&
operator=(_Enable_copy_move const&) noexcept = default;
_Enable_copy_move&
operator=(_Enable_copy_move&&) noexcept = delete;
};
template<typename _Tag>
struct _Enable_copy_move<false, true, false, false, _Tag>
{
constexpr _Enable_copy_move() noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move const&) noexcept = delete;
constexpr _Enable_copy_move(_Enable_copy_move&&) noexcept = delete;
_Enable_copy_move&
operator=(_Enable_copy_move const&) noexcept = default;
_Enable_copy_move&
operator=(_Enable_copy_move&&) noexcept = delete;
};
template<typename _Tag>
struct _Enable_copy_move<true, false, false, false, _Tag>
{
constexpr _Enable_copy_move() noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move const&) noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move&&) noexcept = delete;
_Enable_copy_move&
operator=(_Enable_copy_move const&) noexcept = delete;
_Enable_copy_move&
operator=(_Enable_copy_move&&) noexcept = delete;
};
template<typename _Tag>
struct _Enable_copy_move<false, false, false, false, _Tag>
{
constexpr _Enable_copy_move() noexcept = default;
constexpr _Enable_copy_move(_Enable_copy_move const&) noexcept = delete;
constexpr _Enable_copy_move(_Enable_copy_move&&) noexcept = delete;
_Enable_copy_move&
operator=(_Enable_copy_move const&) noexcept = delete;
_Enable_copy_move&
operator=(_Enable_copy_move&&) noexcept = delete;
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Tp>
class optional;
struct nullopt_t
{
enum class _Construct { _Token };
explicit constexpr nullopt_t(_Construct) noexcept { }
};
inline constexpr nullopt_t nullopt { nullopt_t::_Construct::_Token };
template<typename _Fn> struct _Optional_func { _Fn& _M_f; };
class bad_optional_access : public exception
{
public:
bad_optional_access() = default;
virtual ~bad_optional_access() = default;
const char* what() const noexcept override
{ return "bad optional access"; }
};
[[__noreturn__]] inline void
__throw_bad_optional_access()
{ (throw (bad_optional_access())); }
template <typename _Tp>
struct _Optional_payload_base
{
using _Stored_type = remove_const_t<_Tp>;
_Optional_payload_base() = default;
~_Optional_payload_base() = default;
template<typename... _Args>
constexpr
_Optional_payload_base(in_place_t __tag, _Args&&... __args)
: _M_payload(__tag, std::forward<_Args>(__args)...),
_M_engaged(true)
{ }
template<typename _Up, typename... _Args>
constexpr
_Optional_payload_base(std::initializer_list<_Up> __il,
_Args&&... __args)
: _M_payload(__il, std::forward<_Args>(__args)...),
_M_engaged(true)
{ }
constexpr
_Optional_payload_base(bool __engaged,
const _Optional_payload_base& __other)
{
if (__other._M_engaged)
this->_M_construct(__other._M_get());
}
constexpr
_Optional_payload_base(bool __engaged,
_Optional_payload_base&& __other)
{
if (__other._M_engaged)
this->_M_construct(std::move(__other._M_get()));
}
_Optional_payload_base(const _Optional_payload_base&) = default;
_Optional_payload_base(_Optional_payload_base&&) = default;
_Optional_payload_base&
operator=(const _Optional_payload_base&) = default;
_Optional_payload_base&
operator=(_Optional_payload_base&&) = default;
constexpr void
_M_copy_assign(const _Optional_payload_base& __other)
{
if (this->_M_engaged && __other._M_engaged)
this->_M_get() = __other._M_get();
else
{
if (__other._M_engaged)
this->_M_construct(__other._M_get());
else
this->_M_reset();
}
}
constexpr void
_M_move_assign(_Optional_payload_base&& __other)
noexcept(__and_v<is_nothrow_move_constructible<_Tp>,
is_nothrow_move_assignable<_Tp>>)
{
if (this->_M_engaged && __other._M_engaged)
this->_M_get() = std::move(__other._M_get());
else
{
if (__other._M_engaged)
this->_M_construct(std::move(__other._M_get()));
else
this->_M_reset();
}
}
struct _Empty_byte { };
template<typename _Up, bool = is_trivially_destructible_v<_Up>>
union _Storage
{
constexpr _Storage() noexcept : _M_empty() { }
template<typename... _Args>
constexpr
_Storage(in_place_t, _Args&&... __args)
: _M_value(std::forward<_Args>(__args)...)
{ }
template<typename _Vp, typename... _Args>
constexpr
_Storage(std::initializer_list<_Vp> __il, _Args&&... __args)
: _M_value(__il, std::forward<_Args>(__args)...)
{ }
_Empty_byte _M_empty;
_Up _M_value;
};
template<typename _Up>
union _Storage<_Up, false>
{
constexpr _Storage() noexcept : _M_empty() { }
template<typename... _Args>
constexpr
_Storage(in_place_t, _Args&&... __args)
: _M_value(std::forward<_Args>(__args)...)
{ }
template<typename _Vp, typename... _Args>
constexpr
_Storage(std::initializer_list<_Vp> __il, _Args&&... __args)
: _M_value(__il, std::forward<_Args>(__args)...)
{ }
~_Storage() { }
_Empty_byte _M_empty;
_Up _M_value;
};
_Storage<_Stored_type> _M_payload;
bool _M_engaged = false;
template<typename... _Args>
constexpr void
_M_construct(_Args&&... __args)
noexcept(is_nothrow_constructible_v<_Stored_type, _Args...>)
{
std::_Construct(std::__addressof(this->_M_payload._M_value),
std::forward<_Args>(__args)...);
this->_M_engaged = true;
}
constexpr void
_M_destroy() noexcept
{
_M_engaged = false;
_M_payload._M_value.~_Stored_type();
}
constexpr _Tp&
_M_get() noexcept
{ return this->_M_payload._M_value; }
constexpr const _Tp&
_M_get() const noexcept
{ return this->_M_payload._M_value; }
constexpr void
_M_reset() noexcept
{
if (this->_M_engaged)
_M_destroy();
}
};
template <typename _Tp,
bool =
is_trivially_destructible_v<_Tp>,
bool =
is_trivially_copy_assignable_v<_Tp>
&& is_trivially_copy_constructible_v<_Tp>,
bool =
is_trivially_move_assignable_v<_Tp>
&& is_trivially_move_constructible_v<_Tp>>
struct _Optional_payload;
template <typename _Tp>
struct _Optional_payload<_Tp, true, true, true>
: _Optional_payload_base<_Tp>
{
using _Optional_payload_base<_Tp>::_Optional_payload_base;
_Optional_payload() = default;
};
template <typename _Tp>
struct _Optional_payload<_Tp, true, false, true>
: _Optional_payload_base<_Tp>
{
using _Optional_payload_base<_Tp>::_Optional_payload_base;
_Optional_payload() = default;
~_Optional_payload() = default;
_Optional_payload(const _Optional_payload&) = default;
_Optional_payload(_Optional_payload&&) = default;
_Optional_payload& operator=(_Optional_payload&&) = default;
constexpr
_Optional_payload&
operator=(const _Optional_payload& __other)
{
this->_M_copy_assign(__other);
return *this;
}
};
template <typename _Tp>
struct _Optional_payload<_Tp, true, true, false>
: _Optional_payload_base<_Tp>
{
using _Optional_payload_base<_Tp>::_Optional_payload_base;
_Optional_payload() = default;
~_Optional_payload() = default;
_Optional_payload(const _Optional_payload&) = default;
_Optional_payload(_Optional_payload&&) = default;
_Optional_payload& operator=(const _Optional_payload&) = default;
constexpr
_Optional_payload&
operator=(_Optional_payload&& __other)
noexcept(__and_v<is_nothrow_move_constructible<_Tp>,
is_nothrow_move_assignable<_Tp>>)
{
this->_M_move_assign(std::move(__other));
return *this;
}
};
template <typename _Tp>
struct _Optional_payload<_Tp, true, false, false>
: _Optional_payload_base<_Tp>
{
using _Optional_payload_base<_Tp>::_Optional_payload_base;
_Optional_payload() = default;
~_Optional_payload() = default;
_Optional_payload(const _Optional_payload&) = default;
_Optional_payload(_Optional_payload&&) = default;
constexpr
_Optional_payload&
operator=(const _Optional_payload& __other)
{
this->_M_copy_assign(__other);
return *this;
}
constexpr
_Optional_payload&
operator=(_Optional_payload&& __other)
noexcept(__and_v<is_nothrow_move_constructible<_Tp>,
is_nothrow_move_assignable<_Tp>>)
{
this->_M_move_assign(std::move(__other));
return *this;
}
};
template <typename _Tp, bool _Copy, bool _Move>
struct _Optional_payload<_Tp, false, _Copy, _Move>
: _Optional_payload<_Tp, true, false, false>
{
using _Optional_payload<_Tp, true, false, false>::_Optional_payload;
_Optional_payload() = default;
_Optional_payload(const _Optional_payload&) = default;
_Optional_payload(_Optional_payload&&) = default;
_Optional_payload& operator=(const _Optional_payload&) = default;
_Optional_payload& operator=(_Optional_payload&&) = default;
~_Optional_payload() { this->_M_reset(); }
};
template<typename _Tp, typename _Dp>
class _Optional_base_impl
{
protected:
using _Stored_type = remove_const_t<_Tp>;
template<typename... _Args>
constexpr void
_M_construct(_Args&&... __args)
noexcept(is_nothrow_constructible_v<_Stored_type, _Args...>)
{
static_cast<_Dp*>(this)->_M_payload._M_construct(
std::forward<_Args>(__args)...);
}
constexpr void
_M_destruct() noexcept
{ static_cast<_Dp*>(this)->_M_payload._M_destroy(); }
constexpr void
_M_reset() noexcept
{ static_cast<_Dp*>(this)->_M_payload._M_reset(); }
constexpr bool _M_is_engaged() const noexcept
{ return static_cast<const _Dp*>(this)->_M_payload._M_engaged; }
constexpr _Tp&
_M_get() noexcept
{
do { if (std::__is_constant_evaluated() && !bool(this->_M_is_engaged())) __builtin_unreachable(); } while (false);
return static_cast<_Dp*>(this)->_M_payload._M_get();
}
constexpr const _Tp&
_M_get() const noexcept
{
do { if (std::__is_constant_evaluated() && !bool(this->_M_is_engaged())) __builtin_unreachable(); } while (false);
return static_cast<const _Dp*>(this)->_M_payload._M_get();
}
};
template<typename _Tp,
bool = is_trivially_copy_constructible_v<_Tp>,
bool = is_trivially_move_constructible_v<_Tp>>
struct _Optional_base
: _Optional_base_impl<_Tp, _Optional_base<_Tp>>
{
constexpr _Optional_base() = default;
template<typename... _Args,
enable_if_t<is_constructible_v<_Tp, _Args...>, bool> = false>
constexpr explicit
_Optional_base(in_place_t, _Args&&... __args)
: _M_payload(in_place, std::forward<_Args>(__args)...)
{ }
template<typename _Up, typename... _Args,
enable_if_t<is_constructible_v<_Tp,
initializer_list<_Up>&,
_Args...>, bool> = false>
constexpr explicit
_Optional_base(in_place_t,
initializer_list<_Up> __il,
_Args&&... __args)
: _M_payload(in_place, __il, std::forward<_Args>(__args)...)
{ }
constexpr
_Optional_base(const _Optional_base& __other)
: _M_payload(__other._M_payload._M_engaged, __other._M_payload)
{ }
constexpr
_Optional_base(_Optional_base&& __other)
noexcept(is_nothrow_move_constructible_v<_Tp>)
: _M_payload(__other._M_payload._M_engaged,
std::move(__other._M_payload))
{ }
_Optional_base& operator=(const _Optional_base&) = default;
_Optional_base& operator=(_Optional_base&&) = default;
_Optional_payload<_Tp> _M_payload;
};
template<typename _Tp>
struct _Optional_base<_Tp, false, true>
: _Optional_base_impl<_Tp, _Optional_base<_Tp>>
{
constexpr _Optional_base() = default;
template<typename... _Args,
enable_if_t<is_constructible_v<_Tp, _Args...>, bool> = false>
constexpr explicit
_Optional_base(in_place_t, _Args&&... __args)
: _M_payload(in_place, std::forward<_Args>(__args)...)
{ }
template<typename _Up, typename... _Args,
enable_if_t<is_constructible_v<_Tp,
initializer_list<_Up>&,
_Args...>, bool> = false>
constexpr explicit
_Optional_base(in_place_t,
initializer_list<_Up> __il,
_Args... __args)
: _M_payload(in_place, __il, std::forward<_Args>(__args)...)
{ }
constexpr _Optional_base(const _Optional_base& __other)
: _M_payload(__other._M_payload._M_engaged, __other._M_payload)
{ }
constexpr _Optional_base(_Optional_base&& __other) = default;
_Optional_base& operator=(const _Optional_base&) = default;
_Optional_base& operator=(_Optional_base&&) = default;
_Optional_payload<_Tp> _M_payload;
};
template<typename _Tp>
struct _Optional_base<_Tp, true, false>
: _Optional_base_impl<_Tp, _Optional_base<_Tp>>
{
constexpr _Optional_base() = default;
template<typename... _Args,
enable_if_t<is_constructible_v<_Tp, _Args...>, bool> = false>
constexpr explicit
_Optional_base(in_place_t, _Args&&... __args)
: _M_payload(in_place, std::forward<_Args>(__args)...)
{ }
template<typename _Up, typename... _Args,
enable_if_t<is_constructible_v<_Tp,
initializer_list<_Up>&,
_Args...>, bool> = false>
constexpr explicit
_Optional_base(in_place_t,
initializer_list<_Up> __il,
_Args&&... __args)
: _M_payload(in_place, __il, std::forward<_Args>(__args)...)
{ }
constexpr _Optional_base(const _Optional_base& __other) = default;
constexpr
_Optional_base(_Optional_base&& __other)
noexcept(is_nothrow_move_constructible_v<_Tp>)
: _M_payload(__other._M_payload._M_engaged,
std::move(__other._M_payload))
{ }
_Optional_base& operator=(const _Optional_base&) = default;
_Optional_base& operator=(_Optional_base&&) = default;
_Optional_payload<_Tp> _M_payload;
};
template<typename _Tp>
struct _Optional_base<_Tp, true, true>
: _Optional_base_impl<_Tp, _Optional_base<_Tp>>
{
constexpr _Optional_base() = default;
template<typename... _Args,
enable_if_t<is_constructible_v<_Tp, _Args...>, bool> = false>
constexpr explicit
_Optional_base(in_place_t, _Args&&... __args)
: _M_payload(in_place, std::forward<_Args>(__args)...)
{ }
template<typename _Up, typename... _Args,
enable_if_t<is_constructible_v<_Tp,
initializer_list<_Up>&,
_Args...>, bool> = false>
constexpr explicit
_Optional_base(in_place_t,
initializer_list<_Up> __il,
_Args&&... __args)
: _M_payload(in_place, __il, std::forward<_Args>(__args)...)
{ }
constexpr _Optional_base(const _Optional_base& __other) = default;
constexpr _Optional_base(_Optional_base&& __other) = default;
_Optional_base& operator=(const _Optional_base&) = default;
_Optional_base& operator=(_Optional_base&&) = default;
_Optional_payload<_Tp> _M_payload;
};
template<typename _Tp>
class optional;
template<typename _Tp>
inline constexpr bool __is_optional_v = false;
template<typename _Tp>
inline constexpr bool __is_optional_v<optional<_Tp>> = true;
template<typename _Tp, typename _Up>
using __converts_from_optional =
__or_<is_constructible<_Tp, const optional<_Up>&>,
is_constructible<_Tp, optional<_Up>&>,
is_constructible<_Tp, const optional<_Up>&&>,
is_constructible<_Tp, optional<_Up>&&>,
is_convertible<const optional<_Up>&, _Tp>,
is_convertible<optional<_Up>&, _Tp>,
is_convertible<const optional<_Up>&&, _Tp>,
is_convertible<optional<_Up>&&, _Tp>>;
template<typename _Tp, typename _Up>
using __assigns_from_optional =
__or_<is_assignable<_Tp&, const optional<_Up>&>,
is_assignable<_Tp&, optional<_Up>&>,
is_assignable<_Tp&, const optional<_Up>&&>,
is_assignable<_Tp&, optional<_Up>&&>>;
template<typename _Tp>
class optional
: private _Optional_base<_Tp>,
private _Enable_copy_move<
is_copy_constructible_v<_Tp>,
__and_v<is_copy_constructible<_Tp>, is_copy_assignable<_Tp>>,
is_move_constructible_v<_Tp>,
__and_v<is_move_constructible<_Tp>, is_move_assignable<_Tp>>,
optional<_Tp>>
{
static_assert(!is_same_v<remove_cv_t<_Tp>, nullopt_t>);
static_assert(!is_same_v<remove_cv_t<_Tp>, in_place_t>);
static_assert(is_object_v<_Tp> && !is_array_v<_Tp>);
private:
using _Base = _Optional_base<_Tp>;
template<typename _Up>
using __not_self = __not_<is_same<optional, __remove_cvref_t<_Up>>>;
template<typename _Up>
using __not_tag = __not_<is_same<in_place_t, __remove_cvref_t<_Up>>>;
template<typename... _Cond>
using _Requires = enable_if_t<__and_v<_Cond...>, bool>;
public:
using value_type = _Tp;
constexpr optional() noexcept { }
constexpr optional(nullopt_t) noexcept { }
template<typename _Up = _Tp,
_Requires<__not_self<_Up>, __not_tag<_Up>,
is_constructible<_Tp, _Up>,
is_convertible<_Up, _Tp>> = true>
constexpr
optional(_Up&& __t)
noexcept(is_nothrow_constructible_v<_Tp, _Up>)
: _Base(std::in_place, std::forward<_Up>(__t)) { }
template<typename _Up = _Tp,
_Requires<__not_self<_Up>, __not_tag<_Up>,
is_constructible<_Tp, _Up>,
__not_<is_convertible<_Up, _Tp>>> = false>
explicit constexpr
optional(_Up&& __t)
noexcept(is_nothrow_constructible_v<_Tp, _Up>)
: _Base(std::in_place, std::forward<_Up>(__t)) { }
template<typename _Up,
_Requires<__not_<is_same<_Tp, _Up>>,
is_constructible<_Tp, const _Up&>,
is_convertible<const _Up&, _Tp>,
__not_<__converts_from_optional<_Tp, _Up>>> = true>
constexpr
optional(const optional<_Up>& __t)
noexcept(is_nothrow_constructible_v<_Tp, const _Up&>)
{
if (__t)
emplace(*__t);
}
template<typename _Up,
_Requires<__not_<is_same<_Tp, _Up>>,
is_constructible<_Tp, const _Up&>,
__not_<is_convertible<const _Up&, _Tp>>,
__not_<__converts_from_optional<_Tp, _Up>>> = false>
explicit constexpr
optional(const optional<_Up>& __t)
noexcept(is_nothrow_constructible_v<_Tp, const _Up&>)
{
if (__t)
emplace(*__t);
}
template<typename _Up,
_Requires<__not_<is_same<_Tp, _Up>>,
is_constructible<_Tp, _Up>,
is_convertible<_Up, _Tp>,
__not_<__converts_from_optional<_Tp, _Up>>> = true>
constexpr
optional(optional<_Up>&& __t)
noexcept(is_nothrow_constructible_v<_Tp, _Up>)
{
if (__t)
emplace(std::move(*__t));
}
template<typename _Up,
_Requires<__not_<is_same<_Tp, _Up>>,
is_constructible<_Tp, _Up>,
__not_<is_convertible<_Up, _Tp>>,
__not_<__converts_from_optional<_Tp, _Up>>> = false>
explicit constexpr
optional(optional<_Up>&& __t)
noexcept(is_nothrow_constructible_v<_Tp, _Up>)
{
if (__t)
emplace(std::move(*__t));
}
template<typename... _Args,
_Requires<is_constructible<_Tp, _Args...>> = false>
explicit constexpr
optional(in_place_t, _Args&&... __args)
noexcept(is_nothrow_constructible_v<_Tp, _Args...>)
: _Base(std::in_place, std::forward<_Args>(__args)...) { }
template<typename _Up, typename... _Args,
_Requires<is_constructible<_Tp,
initializer_list<_Up>&,
_Args...>> = false>
explicit constexpr
optional(in_place_t, initializer_list<_Up> __il, _Args&&... __args)
noexcept(is_nothrow_constructible_v<_Tp, initializer_list<_Up>&,
_Args...>)
: _Base(std::in_place, __il, std::forward<_Args>(__args)...) { }
optional&
operator=(nullopt_t) noexcept
{
this->_M_reset();
return *this;
}
template<typename _Up = _Tp>
enable_if_t<__and_v<__not_self<_Up>,
__not_<__and_<is_scalar<_Tp>,
is_same<_Tp, decay_t<_Up>>>>,
is_constructible<_Tp, _Up>,
is_assignable<_Tp&, _Up>>,
optional&>
operator=(_Up&& __u)
noexcept(__and_v<is_nothrow_constructible<_Tp, _Up>,
is_nothrow_assignable<_Tp&, _Up>>)
{
if (this->_M_is_engaged())
this->_M_get() = std::forward<_Up>(__u);
else
this->_M_construct(std::forward<_Up>(__u));
return *this;
}
template<typename _Up>
enable_if_t<__and_v<__not_<is_same<_Tp, _Up>>,
is_constructible<_Tp, const _Up&>,
is_assignable<_Tp&, const _Up&>,
__not_<__converts_from_optional<_Tp, _Up>>,
__not_<__assigns_from_optional<_Tp, _Up>>>,
optional&>
operator=(const optional<_Up>& __u)
noexcept(__and_v<is_nothrow_constructible<_Tp, const _Up&>,
is_nothrow_assignable<_Tp&, const _Up&>>)
{
if (__u)
{
if (this->_M_is_engaged())
this->_M_get() = *__u;
else
this->_M_construct(*__u);
}
else
{
this->_M_reset();
}
return *this;
}
template<typename _Up>
enable_if_t<__and_v<__not_<is_same<_Tp, _Up>>,
is_constructible<_Tp, _Up>,
is_assignable<_Tp&, _Up>,
__not_<__converts_from_optional<_Tp, _Up>>,
__not_<__assigns_from_optional<_Tp, _Up>>>,
optional&>
operator=(optional<_Up>&& __u)
noexcept(__and_v<is_nothrow_constructible<_Tp, _Up>,
is_nothrow_assignable<_Tp&, _Up>>)
{
if (__u)
{
if (this->_M_is_engaged())
this->_M_get() = std::move(*__u);
else
this->_M_construct(std::move(*__u));
}
else
{
this->_M_reset();
}
return *this;
}
template<typename... _Args>
enable_if_t<is_constructible_v<_Tp, _Args...>, _Tp&>
emplace(_Args&&... __args)
noexcept(is_nothrow_constructible_v<_Tp, _Args...>)
{
this->_M_reset();
this->_M_construct(std::forward<_Args>(__args)...);
return this->_M_get();
}
template<typename _Up, typename... _Args>
enable_if_t<is_constructible_v<_Tp, initializer_list<_Up>&, _Args...>,
_Tp&>
emplace(initializer_list<_Up> __il, _Args&&... __args)
noexcept(is_nothrow_constructible_v<_Tp, initializer_list<_Up>&,
_Args...>)
{
this->_M_reset();
this->_M_construct(__il, std::forward<_Args>(__args)...);
return this->_M_get();
}
void
swap(optional& __other)
noexcept(is_nothrow_move_constructible_v<_Tp>
&& is_nothrow_swappable_v<_Tp>)
{
using std::swap;
if (this->_M_is_engaged() && __other._M_is_engaged())
swap(this->_M_get(), __other._M_get());
else if (this->_M_is_engaged())
{
__other._M_construct(std::move(this->_M_get()));
this->_M_destruct();
}
else if (__other._M_is_engaged())
{
this->_M_construct(std::move(__other._M_get()));
__other._M_destruct();
}
}
constexpr const _Tp*
operator->() const noexcept
{ return std::__addressof(this->_M_get()); }
constexpr _Tp*
operator->() noexcept
{ return std::__addressof(this->_M_get()); }
constexpr const _Tp&
operator*() const& noexcept
{ return this->_M_get(); }
constexpr _Tp&
operator*()& noexcept
{ return this->_M_get(); }
constexpr _Tp&&
operator*()&& noexcept
{ return std::move(this->_M_get()); }
constexpr const _Tp&&
operator*() const&& noexcept
{ return std::move(this->_M_get()); }
constexpr explicit operator bool() const noexcept
{ return this->_M_is_engaged(); }
constexpr bool has_value() const noexcept
{ return this->_M_is_engaged(); }
constexpr const _Tp&
value() const&
{
if (this->_M_is_engaged())
return this->_M_get();
__throw_bad_optional_access();
}
constexpr _Tp&
value()&
{
if (this->_M_is_engaged())
return this->_M_get();
__throw_bad_optional_access();
}
constexpr _Tp&&
value()&&
{
if (this->_M_is_engaged())
return std::move(this->_M_get());
__throw_bad_optional_access();
}
constexpr const _Tp&&
value() const&&
{
if (this->_M_is_engaged())
return std::move(this->_M_get());
__throw_bad_optional_access();
}
template<typename _Up>
constexpr _Tp
value_or(_Up&& __u) const&
{
static_assert(is_copy_constructible_v<_Tp>);
static_assert(is_convertible_v<_Up&&, _Tp>);
if (this->_M_is_engaged())
return this->_M_get();
else
return static_cast<_Tp>(std::forward<_Up>(__u));
}
template<typename _Up>
constexpr _Tp
value_or(_Up&& __u) &&
{
static_assert(is_move_constructible_v<_Tp>);
static_assert(is_convertible_v<_Up&&, _Tp>);
if (this->_M_is_engaged())
return std::move(this->_M_get());
else
return static_cast<_Tp>(std::forward<_Up>(__u));
}
void reset() noexcept { this->_M_reset(); }
private:
};
template<typename _Tp>
using __optional_relop_t =
enable_if_t<is_convertible<_Tp, bool>::value, bool>;
template<typename _Tp, typename _Up>
using __optional_eq_t = __optional_relop_t<
decltype(std::declval<const _Tp&>() == std::declval<const _Up&>())
>;
template<typename _Tp, typename _Up>
using __optional_ne_t = __optional_relop_t<
decltype(std::declval<const _Tp&>() != std::declval<const _Up&>())
>;
template<typename _Tp, typename _Up>
using __optional_lt_t = __optional_relop_t<
decltype(std::declval<const _Tp&>() < std::declval<const _Up&>())
>;
template<typename _Tp, typename _Up>
using __optional_gt_t = __optional_relop_t<
decltype(std::declval<const _Tp&>() > std::declval<const _Up&>())
>;
template<typename _Tp, typename _Up>
using __optional_le_t = __optional_relop_t<
decltype(std::declval<const _Tp&>() <= std::declval<const _Up&>())
>;
template<typename _Tp, typename _Up>
using __optional_ge_t = __optional_relop_t<
decltype(std::declval<const _Tp&>() >= std::declval<const _Up&>())
>;
template<typename _Tp, typename _Up>
constexpr auto
operator==(const optional<_Tp>& __lhs, const optional<_Up>& __rhs)
-> __optional_eq_t<_Tp, _Up>
{
return static_cast<bool>(__lhs) == static_cast<bool>(__rhs)
&& (!__lhs || *__lhs == *__rhs);
}
template<typename _Tp, typename _Up>
constexpr auto
operator!=(const optional<_Tp>& __lhs, const optional<_Up>& __rhs)
-> __optional_ne_t<_Tp, _Up>
{
return static_cast<bool>(__lhs) != static_cast<bool>(__rhs)
|| (static_cast<bool>(__lhs) && *__lhs != *__rhs);
}
template<typename _Tp, typename _Up>
constexpr auto
operator<(const optional<_Tp>& __lhs, const optional<_Up>& __rhs)
-> __optional_lt_t<_Tp, _Up>
{
return static_cast<bool>(__rhs) && (!__lhs || *__lhs < *__rhs);
}
template<typename _Tp, typename _Up>
constexpr auto
operator>(const optional<_Tp>& __lhs, const optional<_Up>& __rhs)
-> __optional_gt_t<_Tp, _Up>
{
return static_cast<bool>(__lhs) && (!__rhs || *__lhs > *__rhs);
}
template<typename _Tp, typename _Up>
constexpr auto
operator<=(const optional<_Tp>& __lhs, const optional<_Up>& __rhs)
-> __optional_le_t<_Tp, _Up>
{
return !__lhs || (static_cast<bool>(__rhs) && *__lhs <= *__rhs);
}
template<typename _Tp, typename _Up>
constexpr auto
operator>=(const optional<_Tp>& __lhs, const optional<_Up>& __rhs)
-> __optional_ge_t<_Tp, _Up>
{
return !__rhs || (static_cast<bool>(__lhs) && *__lhs >= *__rhs);
}
template<typename _Tp>
constexpr bool
operator==(const optional<_Tp>& __lhs, nullopt_t) noexcept
{ return !__lhs; }
template<typename _Tp>
constexpr bool
operator==(nullopt_t, const optional<_Tp>& __rhs) noexcept
{ return !__rhs; }
template<typename _Tp>
constexpr bool
operator!=(const optional<_Tp>& __lhs, nullopt_t) noexcept
{ return static_cast<bool>(__lhs); }
template<typename _Tp>
constexpr bool
operator!=(nullopt_t, const optional<_Tp>& __rhs) noexcept
{ return static_cast<bool>(__rhs); }
template<typename _Tp>
constexpr bool
operator<(const optional<_Tp>& , nullopt_t) noexcept
{ return false; }
template<typename _Tp>
constexpr bool
operator<(nullopt_t, const optional<_Tp>& __rhs) noexcept
{ return static_cast<bool>(__rhs); }
template<typename _Tp>
constexpr bool
operator>(const optional<_Tp>& __lhs, nullopt_t) noexcept
{ return static_cast<bool>(__lhs); }
template<typename _Tp>
constexpr bool
operator>(nullopt_t, const optional<_Tp>& ) noexcept
{ return false; }
template<typename _Tp>
constexpr bool
operator<=(const optional<_Tp>& __lhs, nullopt_t) noexcept
{ return !__lhs; }
template<typename _Tp>
constexpr bool
operator<=(nullopt_t, const optional<_Tp>& ) noexcept
{ return true; }
template<typename _Tp>
constexpr bool
operator>=(const optional<_Tp>& , nullopt_t) noexcept
{ return true; }
template<typename _Tp>
constexpr bool
operator>=(nullopt_t, const optional<_Tp>& __rhs) noexcept
{ return !__rhs; }
template<typename _Tp, typename _Up>
constexpr auto
operator==(const optional<_Tp>& __lhs, const _Up& __rhs)
-> __optional_eq_t<_Tp, _Up>
{ return __lhs && *__lhs == __rhs; }
template<typename _Tp, typename _Up>
constexpr auto
operator==(const _Up& __lhs, const optional<_Tp>& __rhs)
-> __optional_eq_t<_Up, _Tp>
{ return __rhs && __lhs == *__rhs; }
template<typename _Tp, typename _Up>
constexpr auto
operator!=(const optional<_Tp>& __lhs, const _Up& __rhs)
-> __optional_ne_t<_Tp, _Up>
{ return !__lhs || *__lhs != __rhs; }
template<typename _Tp, typename _Up>
constexpr auto
operator!=(const _Up& __lhs, const optional<_Tp>& __rhs)
-> __optional_ne_t<_Up, _Tp>
{ return !__rhs || __lhs != *__rhs; }
template<typename _Tp, typename _Up>
constexpr auto
operator<(const optional<_Tp>& __lhs, const _Up& __rhs)
-> __optional_lt_t<_Tp, _Up>
{ return !__lhs || *__lhs < __rhs; }
template<typename _Tp, typename _Up>
constexpr auto
operator<(const _Up& __lhs, const optional<_Tp>& __rhs)
-> __optional_lt_t<_Up, _Tp>
{ return __rhs && __lhs < *__rhs; }
template<typename _Tp, typename _Up>
constexpr auto
operator>(const optional<_Tp>& __lhs, const _Up& __rhs)
-> __optional_gt_t<_Tp, _Up>
{ return __lhs && *__lhs > __rhs; }
template<typename _Tp, typename _Up>
constexpr auto
operator>(const _Up& __lhs, const optional<_Tp>& __rhs)
-> __optional_gt_t<_Up, _Tp>
{ return !__rhs || __lhs > *__rhs; }
template<typename _Tp, typename _Up>
constexpr auto
operator<=(const optional<_Tp>& __lhs, const _Up& __rhs)
-> __optional_le_t<_Tp, _Up>
{ return !__lhs || *__lhs <= __rhs; }
template<typename _Tp, typename _Up>
constexpr auto
operator<=(const _Up& __lhs, const optional<_Tp>& __rhs)
-> __optional_le_t<_Up, _Tp>
{ return __rhs && __lhs <= *__rhs; }
template<typename _Tp, typename _Up>
constexpr auto
operator>=(const optional<_Tp>& __lhs, const _Up& __rhs)
-> __optional_ge_t<_Tp, _Up>
{ return __lhs && *__lhs >= __rhs; }
template<typename _Tp, typename _Up>
constexpr auto
operator>=(const _Up& __lhs, const optional<_Tp>& __rhs)
-> __optional_ge_t<_Up, _Tp>
{ return !__rhs || __lhs >= *__rhs; }
template<typename _Tp>
inline enable_if_t<is_move_constructible_v<_Tp> && is_swappable_v<_Tp>>
swap(optional<_Tp>& __lhs, optional<_Tp>& __rhs)
noexcept(noexcept(__lhs.swap(__rhs)))
{ __lhs.swap(__rhs); }
template<typename _Tp>
enable_if_t<!(is_move_constructible_v<_Tp> && is_swappable_v<_Tp>)>
swap(optional<_Tp>&, optional<_Tp>&) = delete;
template<typename _Tp>
constexpr
enable_if_t<is_constructible_v<decay_t<_Tp>, _Tp>,
optional<decay_t<_Tp>>>
make_optional(_Tp&& __t)
noexcept(is_nothrow_constructible_v<optional<decay_t<_Tp>>, _Tp>)
{ return optional<decay_t<_Tp>>{ std::forward<_Tp>(__t) }; }
template<typename _Tp, typename... _Args>
constexpr
enable_if_t<is_constructible_v<_Tp, _Args...>,
optional<_Tp>>
make_optional(_Args&&... __args)
noexcept(is_nothrow_constructible_v<_Tp, _Args...>)
{ return optional<_Tp>{ in_place, std::forward<_Args>(__args)... }; }
template<typename _Tp, typename _Up, typename... _Args>
constexpr
enable_if_t<is_constructible_v<_Tp, initializer_list<_Up>&, _Args...>,
optional<_Tp>>
make_optional(initializer_list<_Up> __il, _Args&&... __args)
noexcept(is_nothrow_constructible_v<_Tp, initializer_list<_Up>&, _Args...>)
{ return optional<_Tp>{ in_place, __il, std::forward<_Args>(__args)... }; }
template<typename _Tp, typename _Up = remove_const_t<_Tp>,
bool = __poison_hash<_Up>::__enable_hash_call>
struct __optional_hash_call_base
{
size_t
operator()(const optional<_Tp>& __t) const
noexcept(noexcept(hash<_Up>{}(*__t)))
{
constexpr size_t __magic_disengaged_hash = static_cast<size_t>(-3333);
return __t ? hash<_Up>{}(*__t) : __magic_disengaged_hash;
}
};
template<typename _Tp, typename _Up>
struct __optional_hash_call_base<_Tp, _Up, false> {};
template<typename _Tp>
struct hash<optional<_Tp>>
: private __poison_hash<remove_const_t<_Tp>>,
public __optional_hash_call_base<_Tp>
{
using result_type [[__deprecated__]] = size_t;
using argument_type [[__deprecated__]] = optional<_Tp>;
};
template<typename _Tp>
struct __is_fast_hash<hash<optional<_Tp>>> : __is_fast_hash<hash<_Tp>>
{ };
template <typename _Tp> optional(_Tp) -> optional<_Tp>;
}
namespace testing {
namespace internal {
template <typename T>
using Optional = ::std::optional<T>;
inline ::std::nullopt_t Nullopt() { return ::std::nullopt; }
}
}
namespace testing {
namespace internal {
using StringView = ::std::string_view;
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename... _Types> class tuple;
template<typename... _Types> class variant;
template <typename> struct hash;
template<typename _Variant>
struct variant_size;
template<typename _Variant>
struct variant_size<const _Variant> : variant_size<_Variant> {};
template<typename _Variant>
struct variant_size<volatile _Variant> : variant_size<_Variant> {};
template<typename _Variant>
struct variant_size<const volatile _Variant> : variant_size<_Variant> {};
template<typename... _Types>
struct variant_size<variant<_Types...>>
: std::integral_constant<size_t, sizeof...(_Types)> {};
template<typename _Variant>
inline constexpr size_t variant_size_v = variant_size<_Variant>::value;
template<typename... _Types>
inline constexpr size_t
variant_size_v<variant<_Types...>> = sizeof...(_Types);
template<typename... _Types>
inline constexpr size_t
variant_size_v<const variant<_Types...>> = sizeof...(_Types);
template<size_t _Np, typename _Variant>
struct variant_alternative;
template<size_t _Np, typename... _Types>
struct variant_alternative<_Np, variant<_Types...>>
{
static_assert(_Np < sizeof...(_Types));
using type = typename _Nth_type<_Np, _Types...>::type;
};
template<size_t _Np, typename _Variant>
using variant_alternative_t =
typename variant_alternative<_Np, _Variant>::type;
template<size_t _Np, typename _Variant>
struct variant_alternative<_Np, const _Variant>
{ using type = const variant_alternative_t<_Np, _Variant>; };
template<size_t _Np, typename _Variant>
struct variant_alternative<_Np, volatile _Variant>
{ using type = volatile variant_alternative_t<_Np, _Variant>; };
template<size_t _Np, typename _Variant>
struct variant_alternative<_Np, const volatile _Variant>
{ using type = const volatile variant_alternative_t<_Np, _Variant>; };
inline constexpr size_t variant_npos = -1;
template<size_t _Np, typename... _Types>
constexpr variant_alternative_t<_Np, variant<_Types...>>&
get(variant<_Types...>&);
template<size_t _Np, typename... _Types>
constexpr variant_alternative_t<_Np, variant<_Types...>>&&
get(variant<_Types...>&&);
template<size_t _Np, typename... _Types>
constexpr variant_alternative_t<_Np, variant<_Types...>> const&
get(const variant<_Types...>&);
template<size_t _Np, typename... _Types>
constexpr variant_alternative_t<_Np, variant<_Types...>> const&&
get(const variant<_Types...>&&);
template<typename _Result_type, typename _Visitor, typename... _Variants>
constexpr decltype(auto)
__do_visit(_Visitor&& __visitor, _Variants&&... __variants);
template <typename... _Types, typename _Tp>
decltype(auto)
__variant_cast(_Tp&& __rhs)
{
if constexpr (is_lvalue_reference_v<_Tp>)
{
if constexpr (is_const_v<remove_reference_t<_Tp>>)
return static_cast<const variant<_Types...>&>(__rhs);
else
return static_cast<variant<_Types...>&>(__rhs);
}
else
return static_cast<variant<_Types...>&&>(__rhs);
}
namespace __detail
{
namespace __variant
{
struct __variant_cookie {};
struct __variant_idx_cookie { using type = __variant_idx_cookie; };
template<typename _Tp> struct __deduce_visit_result { using type = _Tp; };
template<typename _Visitor, typename... _Variants>
constexpr void
__raw_visit(_Visitor&& __visitor, _Variants&&... __variants)
{
std::__do_visit<__variant_cookie>(std::forward<_Visitor>(__visitor),
std::forward<_Variants>(__variants)...);
}
template<typename _Visitor, typename... _Variants>
constexpr void
__raw_idx_visit(_Visitor&& __visitor, _Variants&&... __variants)
{
std::__do_visit<__variant_idx_cookie>(std::forward<_Visitor>(__visitor),
std::forward<_Variants>(__variants)...);
}
template<typename... _Types>
constexpr std::variant<_Types...>&
__as(std::variant<_Types...>& __v) noexcept
{ return __v; }
template<typename... _Types>
constexpr const std::variant<_Types...>&
__as(const std::variant<_Types...>& __v) noexcept
{ return __v; }
template<typename... _Types>
constexpr std::variant<_Types...>&&
__as(std::variant<_Types...>&& __v) noexcept
{ return std::move(__v); }
template<typename... _Types>
constexpr const std::variant<_Types...>&&
__as(const std::variant<_Types...>&& __v) noexcept
{ return std::move(__v); }
template<typename _Type, bool = std::is_trivially_destructible_v<_Type>>
struct _Uninitialized;
template<typename _Type>
struct _Uninitialized<_Type, true>
{
template<typename... _Args>
constexpr
_Uninitialized(in_place_index_t<0>, _Args&&... __args)
: _M_storage(std::forward<_Args>(__args)...)
{ }
constexpr const _Type& _M_get() const & noexcept
{ return _M_storage; }
constexpr _Type& _M_get() & noexcept
{ return _M_storage; }
constexpr const _Type&& _M_get() const && noexcept
{ return std::move(_M_storage); }
constexpr _Type&& _M_get() && noexcept
{ return std::move(_M_storage); }
_Type _M_storage;
};
template<typename _Type>
struct _Uninitialized<_Type, false>
{
template<typename... _Args>
constexpr
_Uninitialized(in_place_index_t<0>, _Args&&... __args)
{
::new ((void*)std::addressof(_M_storage))
_Type(std::forward<_Args>(__args)...);
}
const _Type& _M_get() const & noexcept
{ return *_M_storage._M_ptr(); }
_Type& _M_get() & noexcept
{ return *_M_storage._M_ptr(); }
const _Type&& _M_get() const && noexcept
{ return std::move(*_M_storage._M_ptr()); }
_Type&& _M_get() && noexcept
{ return std::move(*_M_storage._M_ptr()); }
__gnu_cxx::__aligned_membuf<_Type> _M_storage;
};
template<size_t _Np, typename _Union>
constexpr decltype(auto)
__get_n(_Union&& __u) noexcept
{
if constexpr (_Np == 0)
return std::forward<_Union>(__u)._M_first._M_get();
else if constexpr (_Np == 1)
return std::forward<_Union>(__u)._M_rest._M_first._M_get();
else if constexpr (_Np == 2)
return std::forward<_Union>(__u)._M_rest._M_rest._M_first._M_get();
else
return __variant::__get_n<_Np - 3>(
std::forward<_Union>(__u)._M_rest._M_rest._M_rest);
}
template<size_t _Np, typename _Variant>
constexpr decltype(auto)
__get(_Variant&& __v) noexcept
{ return __variant::__get_n<_Np>(std::forward<_Variant>(__v)._M_u); }
template<typename... _Types>
struct _Traits
{
static constexpr bool _S_default_ctor =
is_default_constructible_v<typename _Nth_type<0, _Types...>::type>;
static constexpr bool _S_copy_ctor =
(is_copy_constructible_v<_Types> && ...);
static constexpr bool _S_move_ctor =
(is_move_constructible_v<_Types> && ...);
static constexpr bool _S_copy_assign =
_S_copy_ctor
&& (is_copy_assignable_v<_Types> && ...);
static constexpr bool _S_move_assign =
_S_move_ctor
&& (is_move_assignable_v<_Types> && ...);
static constexpr bool _S_trivial_dtor =
(is_trivially_destructible_v<_Types> && ...);
static constexpr bool _S_trivial_copy_ctor =
(is_trivially_copy_constructible_v<_Types> && ...);
static constexpr bool _S_trivial_move_ctor =
(is_trivially_move_constructible_v<_Types> && ...);
static constexpr bool _S_trivial_copy_assign =
_S_trivial_dtor && _S_trivial_copy_ctor
&& (is_trivially_copy_assignable_v<_Types> && ...);
static constexpr bool _S_trivial_move_assign =
_S_trivial_dtor && _S_trivial_move_ctor
&& (is_trivially_move_assignable_v<_Types> && ...);
static constexpr bool _S_nothrow_default_ctor =
is_nothrow_default_constructible_v<
typename _Nth_type<0, _Types...>::type>;
static constexpr bool _S_nothrow_copy_ctor = false;
static constexpr bool _S_nothrow_move_ctor =
(is_nothrow_move_constructible_v<_Types> && ...);
static constexpr bool _S_nothrow_copy_assign = false;
static constexpr bool _S_nothrow_move_assign =
_S_nothrow_move_ctor
&& (is_nothrow_move_assignable_v<_Types> && ...);
};
template<typename... _Types>
union _Variadic_union
{
_Variadic_union() = default;
template<size_t _Np, typename... _Args>
_Variadic_union(in_place_index_t<_Np>, _Args&&...) = delete;
};
template<typename _First, typename... _Rest>
union _Variadic_union<_First, _Rest...>
{
constexpr _Variadic_union() : _M_rest() { }
template<typename... _Args>
constexpr
_Variadic_union(in_place_index_t<0>, _Args&&... __args)
: _M_first(in_place_index<0>, std::forward<_Args>(__args)...)
{ }
template<size_t _Np, typename... _Args>
constexpr
_Variadic_union(in_place_index_t<_Np>, _Args&&... __args)
: _M_rest(in_place_index<_Np-1>, std::forward<_Args>(__args)...)
{ }
_Uninitialized<_First> _M_first;
_Variadic_union<_Rest...> _M_rest;
};
template<typename _Tp>
struct _Never_valueless_alt
: __and_<bool_constant<sizeof(_Tp) <= 256>, is_trivially_copyable<_Tp>>
{ };
template <typename... _Types>
constexpr bool __never_valueless()
{
return _Traits<_Types...>::_S_move_assign
&& (_Never_valueless_alt<_Types>::value && ...);
}
template<bool __trivially_destructible, typename... _Types>
struct _Variant_storage;
template <typename... _Types>
using __select_index =
typename __select_int::_Select_int_base<sizeof...(_Types),
unsigned char,
unsigned short>::type::value_type;
template<typename... _Types>
struct _Variant_storage<false, _Types...>
{
constexpr
_Variant_storage()
: _M_index(static_cast<__index_type>(variant_npos))
{ }
template<size_t _Np, typename... _Args>
constexpr
_Variant_storage(in_place_index_t<_Np>, _Args&&... __args)
: _M_u(in_place_index<_Np>, std::forward<_Args>(__args)...),
_M_index{_Np}
{ }
constexpr void
_M_reset()
{
if (!_M_valid()) [[unlikely]]
return;
std::__do_visit<void>([](auto&& __this_mem) mutable
{
std::_Destroy(std::__addressof(__this_mem));
}, __variant_cast<_Types...>(*this));
_M_index = static_cast<__index_type>(variant_npos);
}
~_Variant_storage()
{ _M_reset(); }
constexpr bool
_M_valid() const noexcept
{
if constexpr (__variant::__never_valueless<_Types...>())
return true;
return this->_M_index != __index_type(variant_npos);
}
_Variadic_union<_Types...> _M_u;
using __index_type = __select_index<_Types...>;
__index_type _M_index;
};
template<typename... _Types>
struct _Variant_storage<true, _Types...>
{
constexpr
_Variant_storage()
: _M_index(static_cast<__index_type>(variant_npos))
{ }
template<size_t _Np, typename... _Args>
constexpr
_Variant_storage(in_place_index_t<_Np>, _Args&&... __args)
: _M_u(in_place_index<_Np>, std::forward<_Args>(__args)...),
_M_index{_Np}
{ }
constexpr void
_M_reset() noexcept
{ _M_index = static_cast<__index_type>(variant_npos); }
constexpr bool
_M_valid() const noexcept
{
if constexpr (__variant::__never_valueless<_Types...>())
return true;
return this->_M_index != static_cast<__index_type>(variant_npos);
}
_Variadic_union<_Types...> _M_u;
using __index_type = __select_index<_Types...>;
__index_type _M_index;
};
template<size_t _Np, bool _Triv, typename... _Types, typename... _Args>
inline void
__emplace(_Variant_storage<_Triv, _Types...>& __v, _Args&&... __args)
{
__v._M_reset();
auto* __addr = std::__addressof(__variant::__get_n<_Np>(__v._M_u));
std::_Construct(__addr, std::forward<_Args>(__args)...);
__v._M_index = _Np;
}
template<typename... _Types>
using _Variant_storage_alias =
_Variant_storage<_Traits<_Types...>::_S_trivial_dtor, _Types...>;
template<bool, typename... _Types>
struct _Copy_ctor_base : _Variant_storage_alias<_Types...>
{
using _Base = _Variant_storage_alias<_Types...>;
using _Base::_Base;
_Copy_ctor_base(const _Copy_ctor_base& __rhs)
noexcept(_Traits<_Types...>::_S_nothrow_copy_ctor)
{
__variant::__raw_idx_visit(
[this](auto&& __rhs_mem, auto __rhs_index) mutable
{
constexpr size_t __j = __rhs_index;
if constexpr (__j != variant_npos)
std::_Construct(std::__addressof(this->_M_u),
in_place_index<__j>, __rhs_mem);
}, __variant_cast<_Types...>(__rhs));
this->_M_index = __rhs._M_index;
}
_Copy_ctor_base(_Copy_ctor_base&&) = default;
_Copy_ctor_base& operator=(const _Copy_ctor_base&) = default;
_Copy_ctor_base& operator=(_Copy_ctor_base&&) = default;
};
template<typename... _Types>
struct _Copy_ctor_base<true, _Types...> : _Variant_storage_alias<_Types...>
{
using _Base = _Variant_storage_alias<_Types...>;
using _Base::_Base;
};
template<typename... _Types>
using _Copy_ctor_alias =
_Copy_ctor_base<_Traits<_Types...>::_S_trivial_copy_ctor, _Types...>;
template<bool, typename... _Types>
struct _Move_ctor_base : _Copy_ctor_alias<_Types...>
{
using _Base = _Copy_ctor_alias<_Types...>;
using _Base::_Base;
_Move_ctor_base(_Move_ctor_base&& __rhs)
noexcept(_Traits<_Types...>::_S_nothrow_move_ctor)
{
__variant::__raw_idx_visit(
[this](auto&& __rhs_mem, auto __rhs_index) mutable
{
constexpr size_t __j = __rhs_index;
if constexpr (__j != variant_npos)
std::_Construct(std::__addressof(this->_M_u),
in_place_index<__j>,
std::forward<decltype(__rhs_mem)>(__rhs_mem));
}, __variant_cast<_Types...>(std::move(__rhs)));
this->_M_index = __rhs._M_index;
}
_Move_ctor_base(const _Move_ctor_base&) = default;
_Move_ctor_base& operator=(const _Move_ctor_base&) = default;
_Move_ctor_base& operator=(_Move_ctor_base&&) = default;
};
template<typename... _Types>
struct _Move_ctor_base<true, _Types...> : _Copy_ctor_alias<_Types...>
{
using _Base = _Copy_ctor_alias<_Types...>;
using _Base::_Base;
};
template<typename... _Types>
using _Move_ctor_alias =
_Move_ctor_base<_Traits<_Types...>::_S_trivial_move_ctor, _Types...>;
template<bool, typename... _Types>
struct _Copy_assign_base : _Move_ctor_alias<_Types...>
{
using _Base = _Move_ctor_alias<_Types...>;
using _Base::_Base;
_Copy_assign_base&
operator=(const _Copy_assign_base& __rhs)
noexcept(_Traits<_Types...>::_S_nothrow_copy_assign)
{
__variant::__raw_idx_visit(
[this](auto&& __rhs_mem, auto __rhs_index) mutable
{
constexpr size_t __j = __rhs_index;
if constexpr (__j == variant_npos)
this->_M_reset();
else if (this->_M_index == __j)
__variant::__get<__j>(*this) = __rhs_mem;
else
{
using _Tj = typename _Nth_type<__j, _Types...>::type;
if constexpr (is_nothrow_copy_constructible_v<_Tj>
|| !is_nothrow_move_constructible_v<_Tj>)
__variant::__emplace<__j>(*this, __rhs_mem);
else
{
using _Variant = variant<_Types...>;
_Variant& __self = __variant_cast<_Types...>(*this);
__self = _Variant(in_place_index<__j>, __rhs_mem);
}
}
}, __variant_cast<_Types...>(__rhs));
return *this;
}
_Copy_assign_base(const _Copy_assign_base&) = default;
_Copy_assign_base(_Copy_assign_base&&) = default;
_Copy_assign_base& operator=(_Copy_assign_base&&) = default;
};
template<typename... _Types>
struct _Copy_assign_base<true, _Types...> : _Move_ctor_alias<_Types...>
{
using _Base = _Move_ctor_alias<_Types...>;
using _Base::_Base;
};
template<typename... _Types>
using _Copy_assign_alias =
_Copy_assign_base<_Traits<_Types...>::_S_trivial_copy_assign, _Types...>;
template<bool, typename... _Types>
struct _Move_assign_base : _Copy_assign_alias<_Types...>
{
using _Base = _Copy_assign_alias<_Types...>;
using _Base::_Base;
_Move_assign_base&
operator=(_Move_assign_base&& __rhs)
noexcept(_Traits<_Types...>::_S_nothrow_move_assign)
{
__variant::__raw_idx_visit(
[this](auto&& __rhs_mem, auto __rhs_index) mutable
{
constexpr size_t __j = __rhs_index;
if constexpr (__j != variant_npos)
{
if (this->_M_index == __j)
__variant::__get<__j>(*this) = std::move(__rhs_mem);
else
{
using _Tj = typename _Nth_type<__j, _Types...>::type;
if constexpr (is_nothrow_move_constructible_v<_Tj>)
__variant::__emplace<__j>(*this, std::move(__rhs_mem));
else
{
using _Variant = variant<_Types...>;
_Variant& __self = __variant_cast<_Types...>(*this);
__self.template emplace<__j>(std::move(__rhs_mem));
}
}
}
else
this->_M_reset();
}, __variant_cast<_Types...>(__rhs));
return *this;
}
_Move_assign_base(const _Move_assign_base&) = default;
_Move_assign_base(_Move_assign_base&&) = default;
_Move_assign_base& operator=(const _Move_assign_base&) = default;
};
template<typename... _Types>
struct _Move_assign_base<true, _Types...> : _Copy_assign_alias<_Types...>
{
using _Base = _Copy_assign_alias<_Types...>;
using _Base::_Base;
};
template<typename... _Types>
using _Move_assign_alias =
_Move_assign_base<_Traits<_Types...>::_S_trivial_move_assign, _Types...>;
template<typename... _Types>
struct _Variant_base : _Move_assign_alias<_Types...>
{
using _Base = _Move_assign_alias<_Types...>;
constexpr
_Variant_base() noexcept(_Traits<_Types...>::_S_nothrow_default_ctor)
: _Variant_base(in_place_index<0>) { }
template<size_t _Np, typename... _Args>
constexpr explicit
_Variant_base(in_place_index_t<_Np> __i, _Args&&... __args)
: _Base(__i, std::forward<_Args>(__args)...)
{ }
_Variant_base(const _Variant_base&) = default;
_Variant_base(_Variant_base&&) = default;
_Variant_base& operator=(const _Variant_base&) = default;
_Variant_base& operator=(_Variant_base&&) = default;
};
template<typename _Tp, typename... _Types>
inline constexpr bool __exactly_once
= std::__find_uniq_type_in_pack<_Tp, _Types...>() < sizeof...(_Types);
template<typename _Ti> struct _Arr { _Ti _M_x[1]; };
template<size_t _Ind, typename _Tp, typename _Ti, typename = void>
struct _Build_FUN
{
void _S_fun() = delete;
};
template<size_t _Ind, typename _Tp, typename _Ti>
struct _Build_FUN<_Ind, _Tp, _Ti,
void_t<decltype(_Arr<_Ti>{{std::declval<_Tp>()}})>>
{
static integral_constant<size_t, _Ind> _S_fun(_Ti);
};
template<typename _Tp, typename _Variant,
typename = make_index_sequence<variant_size_v<_Variant>>>
struct _Build_FUNs;
template<typename _Tp, typename... _Ti, size_t... _Ind>
struct _Build_FUNs<_Tp, variant<_Ti...>, index_sequence<_Ind...>>
: _Build_FUN<_Ind, _Tp, _Ti>...
{
using _Build_FUN<_Ind, _Tp, _Ti>::_S_fun...;
};
template<typename _Tp, typename _Variant>
using _FUN_type
= decltype(_Build_FUNs<_Tp, _Variant>::_S_fun(std::declval<_Tp>()));
template<typename _Tp, typename _Variant, typename = void>
inline constexpr size_t
__accepted_index = variant_npos;
template<typename _Tp, typename _Variant>
inline constexpr size_t
__accepted_index<_Tp, _Variant, void_t<_FUN_type<_Tp, _Variant>>>
= _FUN_type<_Tp, _Variant>::value;
template<typename _Maybe_variant_cookie, typename _Variant,
typename = __remove_cvref_t<_Variant>>
inline constexpr bool
__extra_visit_slot_needed = false;
template<typename _Var, typename... _Types>
inline constexpr bool
__extra_visit_slot_needed<__variant_cookie, _Var, variant<_Types...>>
= !__variant::__never_valueless<_Types...>();
template<typename _Var, typename... _Types>
inline constexpr bool
__extra_visit_slot_needed<__variant_idx_cookie, _Var, variant<_Types...>>
= !__variant::__never_valueless<_Types...>();
template<typename _Tp, size_t... _Dimensions>
struct _Multi_array;
template<typename _Tp>
struct _Multi_array<_Tp>
{
template<typename>
struct __untag_result
: false_type
{ using element_type = _Tp; };
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wignored-qualifiers"
template <typename... _Args>
struct __untag_result<const void(*)(_Args...)>
: false_type
{ using element_type = void(*)(_Args...); };
#pragma GCC diagnostic pop
template <typename... _Args>
struct __untag_result<__variant_cookie(*)(_Args...)>
: false_type
{ using element_type = void(*)(_Args...); };
template <typename... _Args>
struct __untag_result<__variant_idx_cookie(*)(_Args...)>
: false_type
{ using element_type = void(*)(_Args...); };
template <typename _Res, typename... _Args>
struct __untag_result<__deduce_visit_result<_Res>(*)(_Args...)>
: true_type
{ using element_type = _Res(*)(_Args...); };
using __result_is_deduced = __untag_result<_Tp>;
constexpr const typename __untag_result<_Tp>::element_type&
_M_access() const
{ return _M_data; }
typename __untag_result<_Tp>::element_type _M_data;
};
template<typename _Ret,
typename _Visitor,
typename... _Variants,
size_t __first, size_t... __rest>
struct _Multi_array<_Ret(*)(_Visitor, _Variants...), __first, __rest...>
{
static constexpr size_t __index =
sizeof...(_Variants) - sizeof...(__rest) - 1;
using _Variant = typename _Nth_type<__index, _Variants...>::type;
static constexpr int __do_cookie =
__extra_visit_slot_needed<_Ret, _Variant> ? 1 : 0;
using _Tp = _Ret(*)(_Visitor, _Variants...);
template<typename... _Args>
constexpr decltype(auto)
_M_access(size_t __first_index, _Args... __rest_indices) const
{
return _M_arr[__first_index + __do_cookie]
._M_access(__rest_indices...);
}
_Multi_array<_Tp, __rest...> _M_arr[__first + __do_cookie];
};
template<typename _Array_type, typename _Index_seq>
struct __gen_vtable_impl;
template<typename _Result_type, typename _Visitor, size_t... __dimensions,
typename... _Variants, size_t... __indices>
struct __gen_vtable_impl<
_Multi_array<_Result_type (*)(_Visitor, _Variants...), __dimensions...>,
std::index_sequence<__indices...>>
{
using _Next =
remove_reference_t<typename _Nth_type<sizeof...(__indices),
_Variants...>::type>;
using _Array_type =
_Multi_array<_Result_type (*)(_Visitor, _Variants...),
__dimensions...>;
static constexpr _Array_type
_S_apply()
{
_Array_type __vtable{};
_S_apply_all_alts(
__vtable, make_index_sequence<variant_size_v<_Next>>());
return __vtable;
}
template<size_t... __var_indices>
static constexpr void
_S_apply_all_alts(_Array_type& __vtable,
std::index_sequence<__var_indices...>)
{
if constexpr (__extra_visit_slot_needed<_Result_type, _Next>)
(_S_apply_single_alt<true, __var_indices>(
__vtable._M_arr[__var_indices + 1],
&(__vtable._M_arr[0])), ...);
else
(_S_apply_single_alt<false, __var_indices>(
__vtable._M_arr[__var_indices]), ...);
}
template<bool __do_cookie, size_t __index, typename _Tp>
static constexpr void
_S_apply_single_alt(_Tp& __element, _Tp* __cookie_element = nullptr)
{
if constexpr (__do_cookie)
{
__element = __gen_vtable_impl<
_Tp,
std::index_sequence<__indices..., __index>>::_S_apply();
*__cookie_element = __gen_vtable_impl<
_Tp,
std::index_sequence<__indices..., variant_npos>>::_S_apply();
}
else
{
auto __tmp_element = __gen_vtable_impl<
remove_reference_t<decltype(__element)>,
std::index_sequence<__indices..., __index>>::_S_apply();
static_assert(is_same_v<_Tp, decltype(__tmp_element)>,
"std::visit requires the visitor to have the same "
"return type for all alternatives of a variant");
__element = __tmp_element;
}
}
};
template<typename _Result_type, typename _Visitor, typename... _Variants,
size_t... __indices>
struct __gen_vtable_impl<
_Multi_array<_Result_type (*)(_Visitor, _Variants...)>,
std::index_sequence<__indices...>>
{
using _Array_type =
_Multi_array<_Result_type (*)(_Visitor, _Variants...)>;
template<size_t __index, typename _Variant>
static constexpr decltype(auto)
__element_by_index_or_cookie(_Variant&& __var) noexcept
{
if constexpr (__index != variant_npos)
return __variant::__get<__index>(std::forward<_Variant>(__var));
else
return __variant_cookie{};
}
static constexpr decltype(auto)
__visit_invoke(_Visitor&& __visitor, _Variants... __vars)
{
if constexpr (is_same_v<_Result_type, __variant_idx_cookie>)
std::__invoke(std::forward<_Visitor>(__visitor),
__element_by_index_or_cookie<__indices>(
std::forward<_Variants>(__vars))...,
integral_constant<size_t, __indices>()...);
else if constexpr (is_same_v<_Result_type, __variant_cookie>)
std::__invoke(std::forward<_Visitor>(__visitor),
__element_by_index_or_cookie<__indices>(
std::forward<_Variants>(__vars))...);
else if constexpr (_Array_type::__result_is_deduced::value)
return std::__invoke(std::forward<_Visitor>(__visitor),
__element_by_index_or_cookie<__indices>(
std::forward<_Variants>(__vars))...);
else
return std::__invoke_r<_Result_type>(
std::forward<_Visitor>(__visitor),
__variant::__get<__indices>(std::forward<_Variants>(__vars))...);
}
static constexpr auto
_S_apply()
{
if constexpr (_Array_type::__result_is_deduced::value)
{
constexpr bool __visit_ret_type_mismatch =
!is_same_v<typename _Result_type::type,
decltype(__visit_invoke(std::declval<_Visitor>(),
std::declval<_Variants>()...))>;
if constexpr (__visit_ret_type_mismatch)
{
struct __cannot_match {};
return __cannot_match{};
}
else
return _Array_type{&__visit_invoke};
}
else
return _Array_type{&__visit_invoke};
}
};
template<typename _Result_type, typename _Visitor, typename... _Variants>
struct __gen_vtable
{
using _Array_type =
_Multi_array<_Result_type (*)(_Visitor, _Variants...),
variant_size_v<remove_reference_t<_Variants>>...>;
static constexpr _Array_type _S_vtable
= __gen_vtable_impl<_Array_type, std::index_sequence<>>::_S_apply();
};
template<size_t _Np, typename _Tp>
struct _Base_dedup : public _Tp { };
template<typename _Variant, typename __indices>
struct _Variant_hash_base;
template<typename... _Types, size_t... __indices>
struct _Variant_hash_base<variant<_Types...>,
std::index_sequence<__indices...>>
: _Base_dedup<__indices, __poison_hash<remove_const_t<_Types>>>... { };
template<size_t _Np, typename _Variant,
typename _AsV = decltype(__variant::__as(std::declval<_Variant>())),
typename _Tp = variant_alternative_t<_Np, remove_reference_t<_AsV>>>
using __get_t
= __conditional_t<is_lvalue_reference_v<_Variant>, _Tp&, _Tp&&>;
template<typename _Visitor, typename... _Variants>
using __visit_result_t
= invoke_result_t<_Visitor, __get_t<0, _Variants>...>;
template<typename _Tp, typename... _Types>
constexpr inline bool __same_types = (is_same_v<_Tp, _Types> && ...);
template <typename _Visitor, typename _Variant, size_t... _Idxs>
constexpr bool __check_visitor_results(std::index_sequence<_Idxs...>)
{
return __same_types<
invoke_result_t<_Visitor, __get_t<_Idxs, _Variant>>...
>;
}
}
}
template<typename _Tp, typename... _Types>
constexpr bool
holds_alternative(const variant<_Types...>& __v) noexcept
{
static_assert(__detail::__variant::__exactly_once<_Tp, _Types...>,
"T must occur exactly once in alternatives");
return __v.index() == std::__find_uniq_type_in_pack<_Tp, _Types...>();
}
template<typename _Tp, typename... _Types>
constexpr _Tp&
get(variant<_Types...>& __v)
{
static_assert(__detail::__variant::__exactly_once<_Tp, _Types...>,
"T must occur exactly once in alternatives");
static_assert(!is_void_v<_Tp>, "_Tp must not be void");
constexpr size_t __n = std::__find_uniq_type_in_pack<_Tp, _Types...>();
return std::get<__n>(__v);
}
template<typename _Tp, typename... _Types>
constexpr _Tp&&
get(variant<_Types...>&& __v)
{
static_assert(__detail::__variant::__exactly_once<_Tp, _Types...>,
"T must occur exactly once in alternatives");
static_assert(!is_void_v<_Tp>, "_Tp must not be void");
constexpr size_t __n = std::__find_uniq_type_in_pack<_Tp, _Types...>();
return std::get<__n>(std::move(__v));
}
template<typename _Tp, typename... _Types>
constexpr const _Tp&
get(const variant<_Types...>& __v)
{
static_assert(__detail::__variant::__exactly_once<_Tp, _Types...>,
"T must occur exactly once in alternatives");
static_assert(!is_void_v<_Tp>, "_Tp must not be void");
constexpr size_t __n = std::__find_uniq_type_in_pack<_Tp, _Types...>();
return std::get<__n>(__v);
}
template<typename _Tp, typename... _Types>
constexpr const _Tp&&
get(const variant<_Types...>&& __v)
{
static_assert(__detail::__variant::__exactly_once<_Tp, _Types...>,
"T must occur exactly once in alternatives");
static_assert(!is_void_v<_Tp>, "_Tp must not be void");
constexpr size_t __n = std::__find_uniq_type_in_pack<_Tp, _Types...>();
return std::get<__n>(std::move(__v));
}
template<size_t _Np, typename... _Types>
constexpr add_pointer_t<variant_alternative_t<_Np, variant<_Types...>>>
get_if(variant<_Types...>* __ptr) noexcept
{
using _Alternative_type = variant_alternative_t<_Np, variant<_Types...>>;
static_assert(_Np < sizeof...(_Types),
"The index must be in [0, number of alternatives)");
static_assert(!is_void_v<_Alternative_type>, "_Tp must not be void");
if (__ptr && __ptr->index() == _Np)
return std::addressof(__detail::__variant::__get<_Np>(*__ptr));
return nullptr;
}
template<size_t _Np, typename... _Types>
constexpr
add_pointer_t<const variant_alternative_t<_Np, variant<_Types...>>>
get_if(const variant<_Types...>* __ptr) noexcept
{
using _Alternative_type = variant_alternative_t<_Np, variant<_Types...>>;
static_assert(_Np < sizeof...(_Types),
"The index must be in [0, number of alternatives)");
static_assert(!is_void_v<_Alternative_type>, "_Tp must not be void");
if (__ptr && __ptr->index() == _Np)
return std::addressof(__detail::__variant::__get<_Np>(*__ptr));
return nullptr;
}
template<typename _Tp, typename... _Types>
constexpr add_pointer_t<_Tp>
get_if(variant<_Types...>* __ptr) noexcept
{
static_assert(__detail::__variant::__exactly_once<_Tp, _Types...>,
"T must occur exactly once in alternatives");
static_assert(!is_void_v<_Tp>, "_Tp must not be void");
constexpr size_t __n = std::__find_uniq_type_in_pack<_Tp, _Types...>();
return std::get_if<__n>(__ptr);
}
template<typename _Tp, typename... _Types>
constexpr add_pointer_t<const _Tp>
get_if(const variant<_Types...>* __ptr) noexcept
{
static_assert(__detail::__variant::__exactly_once<_Tp, _Types...>,
"T must occur exactly once in alternatives");
static_assert(!is_void_v<_Tp>, "_Tp must not be void");
constexpr size_t __n = std::__find_uniq_type_in_pack<_Tp, _Types...>();
return std::get_if<__n>(__ptr);
}
struct monostate { };
template<typename... _Types> constexpr bool operator <(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) { bool __ret = true; __detail::__variant::__raw_idx_visit( [&__ret, &__lhs] (auto&& __rhs_mem, auto __rhs_index) mutable { if constexpr (__rhs_index != variant_npos) { if (__lhs.index() == __rhs_index) { auto& __this_mem = std::get<__rhs_index>(__lhs); __ret = __this_mem < __rhs_mem; } else __ret = (__lhs.index() + 1) < (__rhs_index + 1); } else __ret = (__lhs.index() + 1) < (__rhs_index + 1); }, __rhs); return __ret; }
template<typename... _Types> constexpr bool operator <=(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) { bool __ret = true; __detail::__variant::__raw_idx_visit( [&__ret, &__lhs] (auto&& __rhs_mem, auto __rhs_index) mutable { if constexpr (__rhs_index != variant_npos) { if (__lhs.index() == __rhs_index) { auto& __this_mem = std::get<__rhs_index>(__lhs); __ret = __this_mem <= __rhs_mem; } else __ret = (__lhs.index() + 1) <= (__rhs_index + 1); } else __ret = (__lhs.index() + 1) <= (__rhs_index + 1); }, __rhs); return __ret; }
template<typename... _Types> constexpr bool operator ==(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) { bool __ret = true; __detail::__variant::__raw_idx_visit( [&__ret, &__lhs] (auto&& __rhs_mem, auto __rhs_index) mutable { if constexpr (__rhs_index != variant_npos) { if (__lhs.index() == __rhs_index) { auto& __this_mem = std::get<__rhs_index>(__lhs); __ret = __this_mem == __rhs_mem; } else __ret = (__lhs.index() + 1) == (__rhs_index + 1); } else __ret = (__lhs.index() + 1) == (__rhs_index + 1); }, __rhs); return __ret; }
template<typename... _Types> constexpr bool operator !=(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) { bool __ret = true; __detail::__variant::__raw_idx_visit( [&__ret, &__lhs] (auto&& __rhs_mem, auto __rhs_index) mutable { if constexpr (__rhs_index != variant_npos) { if (__lhs.index() == __rhs_index) { auto& __this_mem = std::get<__rhs_index>(__lhs); __ret = __this_mem != __rhs_mem; } else __ret = (__lhs.index() + 1) != (__rhs_index + 1); } else __ret = (__lhs.index() + 1) != (__rhs_index + 1); }, __rhs); return __ret; }
template<typename... _Types> constexpr bool operator >=(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) { bool __ret = true; __detail::__variant::__raw_idx_visit( [&__ret, &__lhs] (auto&& __rhs_mem, auto __rhs_index) mutable { if constexpr (__rhs_index != variant_npos) { if (__lhs.index() == __rhs_index) { auto& __this_mem = std::get<__rhs_index>(__lhs); __ret = __this_mem >= __rhs_mem; } else __ret = (__lhs.index() + 1) >= (__rhs_index + 1); } else __ret = (__lhs.index() + 1) >= (__rhs_index + 1); }, __rhs); return __ret; }
template<typename... _Types> constexpr bool operator >(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) { bool __ret = true; __detail::__variant::__raw_idx_visit( [&__ret, &__lhs] (auto&& __rhs_mem, auto __rhs_index) mutable { if constexpr (__rhs_index != variant_npos) { if (__lhs.index() == __rhs_index) { auto& __this_mem = std::get<__rhs_index>(__lhs); __ret = __this_mem > __rhs_mem; } else __ret = (__lhs.index() + 1) > (__rhs_index + 1); } else __ret = (__lhs.index() + 1) > (__rhs_index + 1); }, __rhs); return __ret; }
constexpr bool operator==(monostate, monostate) noexcept { return true; }
constexpr bool operator!=(monostate, monostate) noexcept { return false; }
constexpr bool operator<(monostate, monostate) noexcept { return false; }
constexpr bool operator>(monostate, monostate) noexcept { return false; }
constexpr bool operator<=(monostate, monostate) noexcept { return true; }
constexpr bool operator>=(monostate, monostate) noexcept { return true; }
template<typename _Visitor, typename... _Variants>
constexpr __detail::__variant::__visit_result_t<_Visitor, _Variants...>
visit(_Visitor&&, _Variants&&...);
template<typename... _Types>
inline enable_if_t<(is_move_constructible_v<_Types> && ...)
&& (is_swappable_v<_Types> && ...)>
swap(variant<_Types...>& __lhs, variant<_Types...>& __rhs)
noexcept(noexcept(__lhs.swap(__rhs)))
{ __lhs.swap(__rhs); }
template<typename... _Types>
enable_if_t<!((is_move_constructible_v<_Types> && ...)
&& (is_swappable_v<_Types> && ...))>
swap(variant<_Types...>&, variant<_Types...>&) = delete;
class bad_variant_access : public exception
{
public:
bad_variant_access() noexcept { }
const char* what() const noexcept override
{ return _M_reason; }
private:
bad_variant_access(const char* __reason) noexcept : _M_reason(__reason) { }
const char* _M_reason = "bad variant access";
friend void __throw_bad_variant_access(const char* __what);
};
inline void
__throw_bad_variant_access(const char* __what)
{ (throw (bad_variant_access(__what))); }
inline void
__throw_bad_variant_access(bool __valueless)
{
if (__valueless) [[__unlikely__]]
__throw_bad_variant_access("std::get: variant is valueless");
else
__throw_bad_variant_access("std::get: wrong index for variant");
}
template<typename... _Types>
class variant
: private __detail::__variant::_Variant_base<_Types...>,
private _Enable_default_constructor<
__detail::__variant::_Traits<_Types...>::_S_default_ctor,
variant<_Types...>>,
private _Enable_copy_move<
__detail::__variant::_Traits<_Types...>::_S_copy_ctor,
__detail::__variant::_Traits<_Types...>::_S_copy_assign,
__detail::__variant::_Traits<_Types...>::_S_move_ctor,
__detail::__variant::_Traits<_Types...>::_S_move_assign,
variant<_Types...>>
{
private:
template <typename... _UTypes, typename _Tp>
friend decltype(auto)
__variant_cast(_Tp&&);
static_assert(sizeof...(_Types) > 0,
"variant must have at least one alternative");
static_assert(!(std::is_reference_v<_Types> || ...),
"variant must have no reference alternative");
static_assert(!(std::is_void_v<_Types> || ...),
"variant must have no void alternative");
using _Base = __detail::__variant::_Variant_base<_Types...>;
using _Default_ctor_enabler =
_Enable_default_constructor<
__detail::__variant::_Traits<_Types...>::_S_default_ctor,
variant<_Types...>>;
template<typename _Tp>
static constexpr bool __not_self
= !is_same_v<__remove_cvref_t<_Tp>, variant>;
template<typename _Tp>
static constexpr bool
__exactly_once = __detail::__variant::__exactly_once<_Tp, _Types...>;
template<typename _Tp>
static constexpr size_t __accepted_index
= __detail::__variant::__accepted_index<_Tp, variant>;
template<size_t _Np, typename = enable_if_t<(_Np < sizeof...(_Types))>>
using __to_type = typename _Nth_type<_Np, _Types...>::type;
template<typename _Tp, typename = enable_if_t<__not_self<_Tp>>>
using __accepted_type = __to_type<__accepted_index<_Tp>>;
template<typename _Tp>
static constexpr size_t __index_of
= std::__find_uniq_type_in_pack<_Tp, _Types...>();
using _Traits = __detail::__variant::_Traits<_Types...>;
template<typename _Tp>
struct __is_in_place_tag : false_type { };
template<typename _Tp>
struct __is_in_place_tag<in_place_type_t<_Tp>> : true_type { };
template<size_t _Np>
struct __is_in_place_tag<in_place_index_t<_Np>> : true_type { };
template<typename _Tp>
static constexpr bool __not_in_place_tag
= !__is_in_place_tag<__remove_cvref_t<_Tp>>::value;
public:
variant() = default;
variant(const variant& __rhs) = default;
variant(variant&&) = default;
variant& operator=(const variant&) = default;
variant& operator=(variant&&) = default;
~variant() = default;
template<typename _Tp,
typename = enable_if_t<sizeof...(_Types) != 0>,
typename = enable_if_t<__not_in_place_tag<_Tp>>,
typename _Tj = __accepted_type<_Tp&&>,
typename = enable_if_t<__exactly_once<_Tj>
&& is_constructible_v<_Tj, _Tp>>>
constexpr
variant(_Tp&& __t)
noexcept(is_nothrow_constructible_v<_Tj, _Tp>)
: variant(in_place_index<__accepted_index<_Tp>>,
std::forward<_Tp>(__t))
{ }
template<typename _Tp, typename... _Args,
typename = enable_if_t<__exactly_once<_Tp>
&& is_constructible_v<_Tp, _Args...>>>
constexpr explicit
variant(in_place_type_t<_Tp>, _Args&&... __args)
: variant(in_place_index<__index_of<_Tp>>,
std::forward<_Args>(__args)...)
{ }
template<typename _Tp, typename _Up, typename... _Args,
typename = enable_if_t<__exactly_once<_Tp>
&& is_constructible_v<_Tp,
initializer_list<_Up>&, _Args...>>>
constexpr explicit
variant(in_place_type_t<_Tp>, initializer_list<_Up> __il,
_Args&&... __args)
: variant(in_place_index<__index_of<_Tp>>, __il,
std::forward<_Args>(__args)...)
{ }
template<size_t _Np, typename... _Args,
typename _Tp = __to_type<_Np>,
typename = enable_if_t<is_constructible_v<_Tp, _Args...>>>
constexpr explicit
variant(in_place_index_t<_Np>, _Args&&... __args)
: _Base(in_place_index<_Np>, std::forward<_Args>(__args)...),
_Default_ctor_enabler(_Enable_default_constructor_tag{})
{ }
template<size_t _Np, typename _Up, typename... _Args,
typename _Tp = __to_type<_Np>,
typename = enable_if_t<is_constructible_v<_Tp,
initializer_list<_Up>&,
_Args...>>>
constexpr explicit
variant(in_place_index_t<_Np>, initializer_list<_Up> __il,
_Args&&... __args)
: _Base(in_place_index<_Np>, __il, std::forward<_Args>(__args)...),
_Default_ctor_enabler(_Enable_default_constructor_tag{})
{ }
template<typename _Tp>
enable_if_t<__exactly_once<__accepted_type<_Tp&&>>
&& is_constructible_v<__accepted_type<_Tp&&>, _Tp>
&& is_assignable_v<__accepted_type<_Tp&&>&, _Tp>,
variant&>
operator=(_Tp&& __rhs)
noexcept(is_nothrow_assignable_v<__accepted_type<_Tp&&>&, _Tp>
&& is_nothrow_constructible_v<__accepted_type<_Tp&&>, _Tp>)
{
constexpr auto __index = __accepted_index<_Tp>;
if (index() == __index)
std::get<__index>(*this) = std::forward<_Tp>(__rhs);
else
{
using _Tj = __accepted_type<_Tp&&>;
if constexpr (is_nothrow_constructible_v<_Tj, _Tp>
|| !is_nothrow_move_constructible_v<_Tj>)
this->emplace<__index>(std::forward<_Tp>(__rhs));
else
this->emplace<__index>(_Tj(std::forward<_Tp>(__rhs)));
}
return *this;
}
template<typename _Tp, typename... _Args>
enable_if_t<is_constructible_v<_Tp, _Args...> && __exactly_once<_Tp>,
_Tp&>
emplace(_Args&&... __args)
{
constexpr size_t __index = __index_of<_Tp>;
return this->emplace<__index>(std::forward<_Args>(__args)...);
}
template<typename _Tp, typename _Up, typename... _Args>
enable_if_t<is_constructible_v<_Tp, initializer_list<_Up>&, _Args...>
&& __exactly_once<_Tp>,
_Tp&>
emplace(initializer_list<_Up> __il, _Args&&... __args)
{
constexpr size_t __index = __index_of<_Tp>;
return this->emplace<__index>(__il, std::forward<_Args>(__args)...);
}
template<size_t _Np, typename... _Args>
enable_if_t<is_constructible_v<__to_type<_Np>, _Args...>,
__to_type<_Np>&>
emplace(_Args&&... __args)
{
namespace __variant = std::__detail::__variant;
using type = typename _Nth_type<_Np, _Types...>::type;
if constexpr (is_nothrow_constructible_v<type, _Args...>)
{
__variant::__emplace<_Np>(*this, std::forward<_Args>(__args)...);
}
else if constexpr (is_scalar_v<type>)
{
const type __tmp(std::forward<_Args>(__args)...);
__variant::__emplace<_Np>(*this, __tmp);
}
else if constexpr (__variant::_Never_valueless_alt<type>()
&& _Traits::_S_move_assign)
{
variant __tmp(in_place_index<_Np>,
std::forward<_Args>(__args)...);
*this = std::move(__tmp);
}
else
{
__variant::__emplace<_Np>(*this, std::forward<_Args>(__args)...);
}
return std::get<_Np>(*this);
}
template<size_t _Np, typename _Up, typename... _Args>
enable_if_t<is_constructible_v<__to_type<_Np>,
initializer_list<_Up>&, _Args...>,
__to_type<_Np>&>
emplace(initializer_list<_Up> __il, _Args&&... __args)
{
namespace __variant = std::__detail::__variant;
using type = typename _Nth_type<_Np, _Types...>::type;
if constexpr (is_nothrow_constructible_v<type,
initializer_list<_Up>&,
_Args...>)
{
__variant::__emplace<_Np>(*this, __il,
std::forward<_Args>(__args)...);
}
else if constexpr (__variant::_Never_valueless_alt<type>()
&& _Traits::_S_move_assign)
{
variant __tmp(in_place_index<_Np>, __il,
std::forward<_Args>(__args)...);
*this = std::move(__tmp);
}
else
{
__variant::__emplace<_Np>(*this, __il,
std::forward<_Args>(__args)...);
}
return std::get<_Np>(*this);
}
template<size_t _Np, typename... _Args>
enable_if_t<!(_Np < sizeof...(_Types))> emplace(_Args&&...) = delete;
template<typename _Tp, typename... _Args>
enable_if_t<!__exactly_once<_Tp>> emplace(_Args&&...) = delete;
constexpr bool valueless_by_exception() const noexcept
{ return !this->_M_valid(); }
constexpr size_t index() const noexcept
{
using __index_type = typename _Base::__index_type;
if constexpr (__detail::__variant::__never_valueless<_Types...>())
return this->_M_index;
else if constexpr (sizeof...(_Types) <= __index_type(-1) / 2)
return make_signed_t<__index_type>(this->_M_index);
else
return size_t(__index_type(this->_M_index + 1)) - 1;
}
void
swap(variant& __rhs)
noexcept((__is_nothrow_swappable<_Types>::value && ...)
&& is_nothrow_move_constructible_v<variant>)
{
static_assert((is_move_constructible_v<_Types> && ...));
if (__rhs.valueless_by_exception()) [[__unlikely__]]
{
if (!this->valueless_by_exception()) [[__likely__]]
__rhs.swap(*this);
return;
}
namespace __variant = __detail::__variant;
__variant::__raw_idx_visit(
[this, &__rhs](auto&& __rhs_mem, auto __rhs_index) mutable
{
constexpr size_t __j = __rhs_index;
if constexpr (__j != variant_npos)
{
if (this->index() == __j)
{
using std::swap;
swap(std::get<__j>(*this), __rhs_mem);
}
else
{
auto __tmp(std::move(__rhs_mem));
if constexpr (_Traits::_S_trivial_move_assign)
__rhs = std::move(*this);
else
__variant::__raw_idx_visit(
[&__rhs](auto&& __this_mem, auto __this_index) mutable
{
constexpr size_t __k = __this_index;
if constexpr (__k != variant_npos)
__variant::__emplace<__k>(__rhs,
std::move(__this_mem));
}, *this);
__variant::__emplace<__j>(*this, std::move(__tmp));
}
}
}, __rhs);
}
private:
template<size_t _Np, typename _Vp>
friend constexpr decltype(auto)
__detail::__variant::__get(_Vp&& __v) noexcept;
template<typename... _Tp> friend constexpr bool operator <(const variant<_Tp...>& __lhs, const variant<_Tp...>& __rhs);
template<typename... _Tp> friend constexpr bool operator <=(const variant<_Tp...>& __lhs, const variant<_Tp...>& __rhs);
template<typename... _Tp> friend constexpr bool operator ==(const variant<_Tp...>& __lhs, const variant<_Tp...>& __rhs);
template<typename... _Tp> friend constexpr bool operator !=(const variant<_Tp...>& __lhs, const variant<_Tp...>& __rhs);
template<typename... _Tp> friend constexpr bool operator >=(const variant<_Tp...>& __lhs, const variant<_Tp...>& __rhs);
template<typename... _Tp> friend constexpr bool operator >(const variant<_Tp...>& __lhs, const variant<_Tp...>& __rhs);
};
template<size_t _Np, typename... _Types>
constexpr variant_alternative_t<_Np, variant<_Types...>>&
get(variant<_Types...>& __v)
{
static_assert(_Np < sizeof...(_Types),
"The index must be in [0, number of alternatives)");
if (__v.index() != _Np)
__throw_bad_variant_access(__v.valueless_by_exception());
return __detail::__variant::__get<_Np>(__v);
}
template<size_t _Np, typename... _Types>
constexpr variant_alternative_t<_Np, variant<_Types...>>&&
get(variant<_Types...>&& __v)
{
static_assert(_Np < sizeof...(_Types),
"The index must be in [0, number of alternatives)");
if (__v.index() != _Np)
__throw_bad_variant_access(__v.valueless_by_exception());
return __detail::__variant::__get<_Np>(std::move(__v));
}
template<size_t _Np, typename... _Types>
constexpr const variant_alternative_t<_Np, variant<_Types...>>&
get(const variant<_Types...>& __v)
{
static_assert(_Np < sizeof...(_Types),
"The index must be in [0, number of alternatives)");
if (__v.index() != _Np)
__throw_bad_variant_access(__v.valueless_by_exception());
return __detail::__variant::__get<_Np>(__v);
}
template<size_t _Np, typename... _Types>
constexpr const variant_alternative_t<_Np, variant<_Types...>>&&
get(const variant<_Types...>&& __v)
{
static_assert(_Np < sizeof...(_Types),
"The index must be in [0, number of alternatives)");
if (__v.index() != _Np)
__throw_bad_variant_access(__v.valueless_by_exception());
return __detail::__variant::__get<_Np>(std::move(__v));
}
template<typename _Result_type, typename _Visitor, typename... _Variants>
constexpr decltype(auto)
__do_visit(_Visitor&& __visitor, _Variants&&... __variants)
{
if constexpr (sizeof...(_Variants) == 0)
{
if constexpr (is_void_v<_Result_type>)
return (void) std::forward<_Visitor>(__visitor)();
else
return std::forward<_Visitor>(__visitor)();
}
else
{
constexpr size_t __max = 11;
using _V0 = typename _Nth_type<0, _Variants...>::type;
constexpr auto __n = variant_size_v<remove_reference_t<_V0>>;
if constexpr (sizeof...(_Variants) > 1 || __n > __max)
{
constexpr auto& __vtable = __detail::__variant::__gen_vtable<
_Result_type, _Visitor&&, _Variants&&...>::_S_vtable;
auto __func_ptr = __vtable._M_access(__variants.index()...);
return (*__func_ptr)(std::forward<_Visitor>(__visitor),
std::forward<_Variants>(__variants)...);
}
else
{
_V0& __v0
= [](_V0& __v, ...) -> _V0& { return __v; }(__variants...);
using __detail::__variant::_Multi_array;
using __detail::__variant::__gen_vtable_impl;
using _Ma = _Multi_array<_Result_type (*)(_Visitor&&, _V0&&)>;
switch (__v0.index())
{
case 0: { if constexpr (0 < __n) { return __gen_vtable_impl<_Ma, index_sequence<0>>:: __visit_invoke(std::forward<_Visitor>(__visitor), std::forward<_V0>(__v0)); } else __builtin_unreachable(); }
case 1: { if constexpr (1 < __n) { return __gen_vtable_impl<_Ma, index_sequence<1>>:: __visit_invoke(std::forward<_Visitor>(__visitor), std::forward<_V0>(__v0)); } else __builtin_unreachable(); }
case 2: { if constexpr (2 < __n) { return __gen_vtable_impl<_Ma, index_sequence<2>>:: __visit_invoke(std::forward<_Visitor>(__visitor), std::forward<_V0>(__v0)); } else __builtin_unreachable(); }
case 3: { if constexpr (3 < __n) { return __gen_vtable_impl<_Ma, index_sequence<3>>:: __visit_invoke(std::forward<_Visitor>(__visitor), std::forward<_V0>(__v0)); } else __builtin_unreachable(); }
case 4: { if constexpr (4 < __n) { return __gen_vtable_impl<_Ma, index_sequence<4>>:: __visit_invoke(std::forward<_Visitor>(__visitor), std::forward<_V0>(__v0)); } else __builtin_unreachable(); }
case 5: { if constexpr (5 < __n) { return __gen_vtable_impl<_Ma, index_sequence<5>>:: __visit_invoke(std::forward<_Visitor>(__visitor), std::forward<_V0>(__v0)); } else __builtin_unreachable(); }
case 6: { if constexpr (6 < __n) { return __gen_vtable_impl<_Ma, index_sequence<6>>:: __visit_invoke(std::forward<_Visitor>(__visitor), std::forward<_V0>(__v0)); } else __builtin_unreachable(); }
case 7: { if constexpr (7 < __n) { return __gen_vtable_impl<_Ma, index_sequence<7>>:: __visit_invoke(std::forward<_Visitor>(__visitor), std::forward<_V0>(__v0)); } else __builtin_unreachable(); }
case 8: { if constexpr (8 < __n) { return __gen_vtable_impl<_Ma, index_sequence<8>>:: __visit_invoke(std::forward<_Visitor>(__visitor), std::forward<_V0>(__v0)); } else __builtin_unreachable(); }
case 9: { if constexpr (9 < __n) { return __gen_vtable_impl<_Ma, index_sequence<9>>:: __visit_invoke(std::forward<_Visitor>(__visitor), std::forward<_V0>(__v0)); } else __builtin_unreachable(); }
case 10: { if constexpr (10 < __n) { return __gen_vtable_impl<_Ma, index_sequence<10>>:: __visit_invoke(std::forward<_Visitor>(__visitor), std::forward<_V0>(__v0)); } else __builtin_unreachable(); }
case variant_npos:
using __detail::__variant::__variant_idx_cookie;
using __detail::__variant::__variant_cookie;
if constexpr (is_same_v<_Result_type, __variant_idx_cookie>
|| is_same_v<_Result_type, __variant_cookie>)
{
using _Npos = index_sequence<variant_npos>;
return __gen_vtable_impl<_Ma, _Npos>::
__visit_invoke(std::forward<_Visitor>(__visitor),
std::forward<_V0>(__v0));
}
else
__builtin_unreachable();
default:
__builtin_unreachable();
}
}
}
}
template<typename _Visitor, typename... _Variants>
constexpr __detail::__variant::__visit_result_t<_Visitor, _Variants...>
visit(_Visitor&& __visitor, _Variants&&... __variants)
{
namespace __variant = std::__detail::__variant;
if ((__variant::__as(__variants).valueless_by_exception() || ...))
__throw_bad_variant_access("std::visit: variant is valueless");
using _Result_type
= __detail::__variant::__visit_result_t<_Visitor, _Variants...>;
using _Tag = __detail::__variant::__deduce_visit_result<_Result_type>;
if constexpr (sizeof...(_Variants) == 1)
{
using _Vp = decltype(__variant::__as(std::declval<_Variants>()...));
constexpr bool __visit_rettypes_match = __detail::__variant::
__check_visitor_results<_Visitor, _Vp>(
make_index_sequence<variant_size_v<remove_reference_t<_Vp>>>());
if constexpr (!__visit_rettypes_match)
{
static_assert(__visit_rettypes_match,
"std::visit requires the visitor to have the same "
"return type for all alternatives of a variant");
return;
}
else
return std::__do_visit<_Tag>(
std::forward<_Visitor>(__visitor),
static_cast<_Vp>(__variants)...);
}
else
return std::__do_visit<_Tag>(
std::forward<_Visitor>(__visitor),
__variant::__as(std::forward<_Variants>(__variants))...);
}
template<bool, typename... _Types>
struct __variant_hash_call_base_impl
{
size_t
operator()(const variant<_Types...>& __t) const
noexcept((is_nothrow_invocable_v<hash<decay_t<_Types>>, _Types> && ...))
{
size_t __ret;
__detail::__variant::__raw_visit(
[&__t, &__ret](auto&& __t_mem) mutable
{
using _Type = __remove_cvref_t<decltype(__t_mem)>;
if constexpr (!is_same_v<_Type,
__detail::__variant::__variant_cookie>)
__ret = std::hash<size_t>{}(__t.index())
+ std::hash<_Type>{}(__t_mem);
else
__ret = std::hash<size_t>{}(__t.index());
}, __t);
return __ret;
}
};
template<typename... _Types>
struct __variant_hash_call_base_impl<false, _Types...> {};
template<typename... _Types>
using __variant_hash_call_base =
__variant_hash_call_base_impl<(__poison_hash<remove_const_t<_Types>>::
__enable_hash_call &&...), _Types...>;
template<typename... _Types>
struct hash<variant<_Types...>>
: private __detail::__variant::_Variant_hash_base<
variant<_Types...>, std::index_sequence_for<_Types...>>,
public __variant_hash_call_base<_Types...>
{
using result_type [[__deprecated__]] = size_t;
using argument_type [[__deprecated__]] = variant<_Types...>;
};
template<>
struct hash<monostate>
{
using result_type [[__deprecated__]] = size_t;
using argument_type [[__deprecated__]] = monostate;
size_t
operator()(const monostate&) const noexcept
{
constexpr size_t __magic_monostate_hash = -7777;
return __magic_monostate_hash;
}
};
template<typename... _Types>
struct __is_fast_hash<hash<variant<_Types...>>>
: bool_constant<(__is_fast_hash<_Types>::value && ...)>
{ };
}
namespace testing {
namespace internal {
template <typename... T>
using Variant = ::std::variant<T...>;
}
}
void operator<<(const testing::internal::Secret&, int);
namespace testing {
class __attribute__((visibility("default"))) Message {
private:
typedef std::ostream& (*BasicNarrowIoManip)(std::ostream&);
public:
Message();
Message(const Message& msg) : ss_(new ::std::stringstream) {
*ss_ << msg.GetString();
}
explicit Message(const char* str) : ss_(new ::std::stringstream) {
*ss_ << str;
}
template <typename T>
inline Message& operator<<(const T& val) {
using ::operator<<;
*ss_ << val;
return *this;
}
template <typename T>
inline Message& operator<<(T* const& pointer) {
if (pointer == nullptr) {
*ss_ << "(null)";
} else {
*ss_ << pointer;
}
return *this;
}
Message& operator<<(BasicNarrowIoManip val) {
*ss_ << val;
return *this;
}
Message& operator<<(bool b) { return *this << (b ? "true" : "false"); }
Message& operator<<(const wchar_t* wide_c_str);
Message& operator<<(wchar_t* wide_c_str);
Message& operator<<(const ::std::wstring& wstr);
std::string GetString() const;
private:
const std::unique_ptr< ::std::stringstream> ss_;
void operator=(const Message&);
};
inline std::ostream& operator<<(std::ostream& os, const Message& sb) {
return os << sb.GetString();
}
namespace internal {
template <typename T>
std::string StreamableToString(const T& streamable) {
return (Message() << streamable).GetString();
}
}
}
namespace testing {
class __attribute__((visibility("default"))) AssertionResult {
public:
AssertionResult(const AssertionResult& other);
template <typename T>
explicit AssertionResult(
const T& success,
typename std::enable_if<
!std::is_convertible<T, AssertionResult>::value>::type*
= nullptr)
: success_(success) {}
AssertionResult& operator=(AssertionResult other) {
swap(other);
return *this;
}
operator bool() const { return success_; }
AssertionResult operator!() const;
const char* message() const {
return message_.get() != nullptr ? message_->c_str() : "";
}
const char* failure_message() const { return message(); }
template <typename T>
AssertionResult& operator<<(const T& value) {
AppendMessage(Message() << value);
return *this;
}
AssertionResult& operator<<(
::std::ostream& (*basic_manipulator)(::std::ostream& stream)) {
AppendMessage(Message() << basic_manipulator);
return *this;
}
private:
void AppendMessage(const Message& a_message) {
if (message_.get() == nullptr) message_.reset(new ::std::string);
message_->append(a_message.GetString().c_str());
}
void swap(AssertionResult& other);
bool success_;
std::unique_ptr< ::std::string> message_;
};
__attribute__((visibility("default"))) AssertionResult AssertionSuccess();
__attribute__((visibility("default"))) AssertionResult AssertionFailure();
__attribute__((visibility("default"))) AssertionResult AssertionFailure(const Message& msg);
}
namespace std __attribute__ ((__visibility__ ("default")))
{
class bad_function_call : public std::exception
{
public:
virtual ~bad_function_call() noexcept;
const char* what() const noexcept;
};
template<typename _Tp>
struct __is_location_invariant
: is_trivially_copyable<_Tp>::type
{ };
class _Undefined_class;
union _Nocopy_types
{
void* _M_object;
const void* _M_const_object;
void (*_M_function_pointer)();
void (_Undefined_class::*_M_member_pointer)();
};
union [[gnu::may_alias]] _Any_data
{
void* _M_access() noexcept { return &_M_pod_data[0]; }
const void* _M_access() const noexcept { return &_M_pod_data[0]; }
template<typename _Tp>
_Tp&
_M_access() noexcept
{ return *static_cast<_Tp*>(_M_access()); }
template<typename _Tp>
const _Tp&
_M_access() const noexcept
{ return *static_cast<const _Tp*>(_M_access()); }
_Nocopy_types _M_unused;
char _M_pod_data[sizeof(_Nocopy_types)];
};
enum _Manager_operation
{
__get_type_info,
__get_functor_ptr,
__clone_functor,
__destroy_functor
};
template<typename _Signature>
class function;
class _Function_base
{
public:
static const size_t _M_max_size = sizeof(_Nocopy_types);
static const size_t _M_max_align = __alignof__(_Nocopy_types);
template<typename _Functor>
class _Base_manager
{
protected:
static const bool __stored_locally =
(__is_location_invariant<_Functor>::value
&& sizeof(_Functor) <= _M_max_size
&& __alignof__(_Functor) <= _M_max_align
&& (_M_max_align % __alignof__(_Functor) == 0));
using _Local_storage = integral_constant<bool, __stored_locally>;
static _Functor*
_M_get_pointer(const _Any_data& __source) noexcept
{
if constexpr (__stored_locally)
{
const _Functor& __f = __source._M_access<_Functor>();
return const_cast<_Functor*>(std::__addressof(__f));
}
else
return __source._M_access<_Functor*>();
}
private:
template<typename _Fn>
static void
_M_create(_Any_data& __dest, _Fn&& __f, true_type)
{
::new (__dest._M_access()) _Functor(std::forward<_Fn>(__f));
}
template<typename _Fn>
static void
_M_create(_Any_data& __dest, _Fn&& __f, false_type)
{
__dest._M_access<_Functor*>()
= new _Functor(std::forward<_Fn>(__f));
}
static void
_M_destroy(_Any_data& __victim, true_type)
{
__victim._M_access<_Functor>().~_Functor();
}
static void
_M_destroy(_Any_data& __victim, false_type)
{
delete __victim._M_access<_Functor*>();
}
public:
static bool
_M_manager(_Any_data& __dest, const _Any_data& __source,
_Manager_operation __op)
{
switch (__op)
{
case __get_type_info:
__dest._M_access<const type_info*>() = &typeid(_Functor);
break;
case __get_functor_ptr:
__dest._M_access<_Functor*>() = _M_get_pointer(__source);
break;
case __clone_functor:
_M_init_functor(__dest,
*const_cast<const _Functor*>(_M_get_pointer(__source)));
break;
case __destroy_functor:
_M_destroy(__dest, _Local_storage());
break;
}
return false;
}
template<typename _Fn>
static void
_M_init_functor(_Any_data& __functor, _Fn&& __f)
noexcept(__and_<_Local_storage,
is_nothrow_constructible<_Functor, _Fn>>::value)
{
_M_create(__functor, std::forward<_Fn>(__f), _Local_storage());
}
template<typename _Signature>
static bool
_M_not_empty_function(const function<_Signature>& __f) noexcept
{ return static_cast<bool>(__f); }
template<typename _Tp>
static bool
_M_not_empty_function(_Tp* __fp) noexcept
{ return __fp != nullptr; }
template<typename _Class, typename _Tp>
static bool
_M_not_empty_function(_Tp _Class::* __mp) noexcept
{ return __mp != nullptr; }
template<typename _Tp>
static bool
_M_not_empty_function(const _Tp&) noexcept
{ return true; }
};
_Function_base() = default;
~_Function_base()
{
if (_M_manager)
_M_manager(_M_functor, _M_functor, __destroy_functor);
}
bool _M_empty() const { return !_M_manager; }
using _Manager_type
= bool (*)(_Any_data&, const _Any_data&, _Manager_operation);
_Any_data _M_functor{};
_Manager_type _M_manager{};
};
template<typename _Signature, typename _Functor>
class _Function_handler;
template<typename _Res, typename _Functor, typename... _ArgTypes>
class _Function_handler<_Res(_ArgTypes...), _Functor>
: public _Function_base::_Base_manager<_Functor>
{
using _Base = _Function_base::_Base_manager<_Functor>;
public:
static bool
_M_manager(_Any_data& __dest, const _Any_data& __source,
_Manager_operation __op)
{
switch (__op)
{
case __get_type_info:
__dest._M_access<const type_info*>() = &typeid(_Functor);
break;
case __get_functor_ptr:
__dest._M_access<_Functor*>() = _Base::_M_get_pointer(__source);
break;
default:
_Base::_M_manager(__dest, __source, __op);
}
return false;
}
static _Res
_M_invoke(const _Any_data& __functor, _ArgTypes&&... __args)
{
return std::__invoke_r<_Res>(*_Base::_M_get_pointer(__functor),
std::forward<_ArgTypes>(__args)...);
}
template<typename _Fn>
static constexpr bool
_S_nothrow_init() noexcept
{
return __and_<typename _Base::_Local_storage,
is_nothrow_constructible<_Functor, _Fn>>::value;
}
};
template<>
class _Function_handler<void, void>
{
public:
static bool
_M_manager(_Any_data&, const _Any_data&, _Manager_operation)
{ return false; }
};
template<typename _Signature, typename _Functor,
bool __valid = is_object<_Functor>::value>
struct _Target_handler
: _Function_handler<_Signature, typename remove_cv<_Functor>::type>
{ };
template<typename _Signature, typename _Functor>
struct _Target_handler<_Signature, _Functor, false>
: _Function_handler<void, void>
{ };
template<typename _Res, typename... _ArgTypes>
class function<_Res(_ArgTypes...)>
: public _Maybe_unary_or_binary_function<_Res, _ArgTypes...>,
private _Function_base
{
template<typename _Func,
bool _Self = is_same<__remove_cvref_t<_Func>, function>::value>
using _Decay_t
= typename __enable_if_t<!_Self, decay<_Func>>::type;
template<typename _Func,
typename _DFunc = _Decay_t<_Func>,
typename _Res2 = __invoke_result<_DFunc&, _ArgTypes...>>
struct _Callable
: __is_invocable_impl<_Res2, _Res>::type
{ };
template<typename _Cond, typename _Tp = void>
using _Requires = __enable_if_t<_Cond::value, _Tp>;
template<typename _Functor>
using _Handler
= _Function_handler<_Res(_ArgTypes...), __decay_t<_Functor>>;
public:
typedef _Res result_type;
function() noexcept
: _Function_base() { }
function(nullptr_t) noexcept
: _Function_base() { }
function(const function& __x)
: _Function_base()
{
if (static_cast<bool>(__x))
{
__x._M_manager(_M_functor, __x._M_functor, __clone_functor);
_M_invoker = __x._M_invoker;
_M_manager = __x._M_manager;
}
}
function(function&& __x) noexcept
: _Function_base(), _M_invoker(__x._M_invoker)
{
if (static_cast<bool>(__x))
{
_M_functor = __x._M_functor;
_M_manager = __x._M_manager;
__x._M_manager = nullptr;
__x._M_invoker = nullptr;
}
}
template<typename _Functor,
typename _Constraints = _Requires<_Callable<_Functor>>>
function(_Functor&& __f)
noexcept(_Handler<_Functor>::template _S_nothrow_init<_Functor>())
: _Function_base()
{
static_assert(is_copy_constructible<__decay_t<_Functor>>::value,
"std::function target must be copy-constructible");
static_assert(is_constructible<__decay_t<_Functor>, _Functor>::value,
"std::function target must be constructible from the "
"constructor argument");
using _My_handler = _Handler<_Functor>;
if (_My_handler::_M_not_empty_function(__f))
{
_My_handler::_M_init_functor(_M_functor,
std::forward<_Functor>(__f));
_M_invoker = &_My_handler::_M_invoke;
_M_manager = &_My_handler::_M_manager;
}
}
function&
operator=(const function& __x)
{
function(__x).swap(*this);
return *this;
}
function&
operator=(function&& __x) noexcept
{
function(std::move(__x)).swap(*this);
return *this;
}
function&
operator=(nullptr_t) noexcept
{
if (_M_manager)
{
_M_manager(_M_functor, _M_functor, __destroy_functor);
_M_manager = nullptr;
_M_invoker = nullptr;
}
return *this;
}
template<typename _Functor>
_Requires<_Callable<_Functor>, function&>
operator=(_Functor&& __f)
noexcept(_Handler<_Functor>::template _S_nothrow_init<_Functor>())
{
function(std::forward<_Functor>(__f)).swap(*this);
return *this;
}
template<typename _Functor>
function&
operator=(reference_wrapper<_Functor> __f) noexcept
{
function(__f).swap(*this);
return *this;
}
void swap(function& __x) noexcept
{
std::swap(_M_functor, __x._M_functor);
std::swap(_M_manager, __x._M_manager);
std::swap(_M_invoker, __x._M_invoker);
}
explicit operator bool() const noexcept
{ return !_M_empty(); }
_Res
operator()(_ArgTypes... __args) const
{
if (_M_empty())
__throw_bad_function_call();
return _M_invoker(_M_functor, std::forward<_ArgTypes>(__args)...);
}
const type_info&
target_type() const noexcept
{
if (_M_manager)
{
_Any_data __typeinfo_result;
_M_manager(__typeinfo_result, _M_functor, __get_type_info);
if (auto __ti = __typeinfo_result._M_access<const type_info*>())
return *__ti;
}
return typeid(void);
}
template<typename _Functor>
_Functor*
target() noexcept
{
const function* __const_this = this;
const _Functor* __func = __const_this->template target<_Functor>();
return *const_cast<_Functor**>(&__func);
}
template<typename _Functor>
const _Functor*
target() const noexcept
{
if constexpr (is_object<_Functor>::value)
{
using _Handler = _Target_handler<_Res(_ArgTypes...), _Functor>;
if (_M_manager == &_Handler::_M_manager
|| (_M_manager && typeid(_Functor) == target_type())
)
{
_Any_data __ptr;
_M_manager(__ptr, _M_functor, __get_functor_ptr);
return __ptr._M_access<const _Functor*>();
}
}
return nullptr;
}
private:
using _Invoker_type = _Res (*)(const _Any_data&, _ArgTypes&&...);
_Invoker_type _M_invoker = nullptr;
};
template<typename>
struct __function_guide_helper
{ };
template<typename _Res, typename _Tp, bool _Nx, typename... _Args>
struct __function_guide_helper<
_Res (_Tp::*) (_Args...) noexcept(_Nx)
>
{ using type = _Res(_Args...); };
template<typename _Res, typename _Tp, bool _Nx, typename... _Args>
struct __function_guide_helper<
_Res (_Tp::*) (_Args...) & noexcept(_Nx)
>
{ using type = _Res(_Args...); };
template<typename _Res, typename _Tp, bool _Nx, typename... _Args>
struct __function_guide_helper<
_Res (_Tp::*) (_Args...) const noexcept(_Nx)
>
{ using type = _Res(_Args...); };
template<typename _Res, typename _Tp, bool _Nx, typename... _Args>
struct __function_guide_helper<
_Res (_Tp::*) (_Args...) const & noexcept(_Nx)
>
{ using type = _Res(_Args...); };
template<typename _Fn, typename _Op>
using __function_guide_t = typename __function_guide_helper<_Op>::type;
template<typename _Res, typename... _ArgTypes>
function(_Res(*)(_ArgTypes...)) -> function<_Res(_ArgTypes...)>;
template<typename _Fn, typename _Signature
= __function_guide_t<_Fn, decltype(&_Fn::operator())>>
function(_Fn) -> function<_Signature>;
template<typename _Res, typename... _Args>
inline bool
operator==(const function<_Res(_Args...)>& __f, nullptr_t) noexcept
{ return !static_cast<bool>(__f); }
template<typename _Res, typename... _Args>
inline bool
operator==(nullptr_t, const function<_Res(_Args...)>& __f) noexcept
{ return !static_cast<bool>(__f); }
template<typename _Res, typename... _Args>
inline bool
operator!=(const function<_Res(_Args...)>& __f, nullptr_t) noexcept
{ return static_cast<bool>(__f); }
template<typename _Res, typename... _Args>
inline bool
operator!=(nullptr_t, const function<_Res(_Args...)>& __f) noexcept
{ return static_cast<bool>(__f); }
template<typename _Res, typename... _Args>
inline void
swap(function<_Res(_Args...)>& __x, function<_Res(_Args...)>& __y) noexcept
{ __x.swap(__y); }
namespace __detail::__variant
{
template<typename> struct _Never_valueless_alt;
template<typename _Signature>
struct _Never_valueless_alt<std::function<_Signature>>
: std::true_type
{ };
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
class _Hashtable;
namespace __detail
{
template<typename _Key, typename _Value, typename _ExtractKey,
typename _Equal, typename _Hash, typename _RangeHash,
typename _Unused, typename _Traits>
struct _Hashtable_base;
template<typename _Iterator>
inline typename std::iterator_traits<_Iterator>::difference_type
__distance_fw(_Iterator __first, _Iterator __last,
std::input_iterator_tag)
{ return __first != __last ? 1 : 0; }
template<typename _Iterator>
inline typename std::iterator_traits<_Iterator>::difference_type
__distance_fw(_Iterator __first, _Iterator __last,
std::forward_iterator_tag)
{ return std::distance(__first, __last); }
template<typename _Iterator>
inline typename std::iterator_traits<_Iterator>::difference_type
__distance_fw(_Iterator __first, _Iterator __last)
{ return __distance_fw(__first, __last,
std::__iterator_category(__first)); }
struct _Identity
{
template<typename _Tp>
_Tp&&
operator()(_Tp&& __x) const noexcept
{ return std::forward<_Tp>(__x); }
};
struct _Select1st
{
template<typename _Pair>
struct __1st_type;
template<typename _Tp, typename _Up>
struct __1st_type<pair<_Tp, _Up>>
{ using type = _Tp; };
template<typename _Tp, typename _Up>
struct __1st_type<const pair<_Tp, _Up>>
{ using type = const _Tp; };
template<typename _Pair>
struct __1st_type<_Pair&>
{ using type = typename __1st_type<_Pair>::type&; };
template<typename _Tp>
typename __1st_type<_Tp>::type&&
operator()(_Tp&& __x) const noexcept
{ return std::forward<_Tp>(__x).first; }
};
template<typename _ExKey, typename _Value>
struct _ConvertToValueType;
template<typename _Value>
struct _ConvertToValueType<_Identity, _Value>
{
template<typename _Kt>
constexpr _Kt&&
operator()(_Kt&& __k) const noexcept
{ return std::forward<_Kt>(__k); }
};
template<typename _Value>
struct _ConvertToValueType<_Select1st, _Value>
{
constexpr _Value&&
operator()(_Value&& __x) const noexcept
{ return std::move(__x); }
constexpr const _Value&
operator()(const _Value& __x) const noexcept
{ return __x; }
template<typename _Kt, typename _Val>
constexpr std::pair<_Kt, _Val>&&
operator()(std::pair<_Kt, _Val>&& __x) const noexcept
{ return std::move(__x); }
template<typename _Kt, typename _Val>
constexpr const std::pair<_Kt, _Val>&
operator()(const std::pair<_Kt, _Val>& __x) const noexcept
{ return __x; }
};
template<typename _ExKey>
struct _NodeBuilder;
template<>
struct _NodeBuilder<_Select1st>
{
template<typename _Kt, typename _Arg, typename _NodeGenerator>
static auto
_S_build(_Kt&& __k, _Arg&& __arg, const _NodeGenerator& __node_gen)
-> typename _NodeGenerator::__node_type*
{
return __node_gen(std::forward<_Kt>(__k),
std::forward<_Arg>(__arg).second);
}
};
template<>
struct _NodeBuilder<_Identity>
{
template<typename _Kt, typename _Arg, typename _NodeGenerator>
static auto
_S_build(_Kt&& __k, _Arg&&, const _NodeGenerator& __node_gen)
-> typename _NodeGenerator::__node_type*
{ return __node_gen(std::forward<_Kt>(__k)); }
};
template<typename _NodeAlloc>
struct _Hashtable_alloc;
template<typename _NodeAlloc>
struct _ReuseOrAllocNode
{
private:
using __node_alloc_type = _NodeAlloc;
using __hashtable_alloc = _Hashtable_alloc<__node_alloc_type>;
using __node_alloc_traits =
typename __hashtable_alloc::__node_alloc_traits;
public:
using __node_type = typename __hashtable_alloc::__node_type;
_ReuseOrAllocNode(__node_type* __nodes, __hashtable_alloc& __h)
: _M_nodes(__nodes), _M_h(__h) { }
_ReuseOrAllocNode(const _ReuseOrAllocNode&) = delete;
~_ReuseOrAllocNode()
{ _M_h._M_deallocate_nodes(_M_nodes); }
template<typename... _Args>
__node_type*
operator()(_Args&&... __args) const
{
if (_M_nodes)
{
__node_type* __node = _M_nodes;
_M_nodes = _M_nodes->_M_next();
__node->_M_nxt = nullptr;
auto& __a = _M_h._M_node_allocator();
__node_alloc_traits::destroy(__a, __node->_M_valptr());
try
{
__node_alloc_traits::construct(__a, __node->_M_valptr(),
std::forward<_Args>(__args)...);
}
catch(...)
{
_M_h._M_deallocate_node_ptr(__node);
throw;
}
return __node;
}
return _M_h._M_allocate_node(std::forward<_Args>(__args)...);
}
private:
mutable __node_type* _M_nodes;
__hashtable_alloc& _M_h;
};
template<typename _NodeAlloc>
struct _AllocNode
{
private:
using __hashtable_alloc = _Hashtable_alloc<_NodeAlloc>;
public:
using __node_type = typename __hashtable_alloc::__node_type;
_AllocNode(__hashtable_alloc& __h)
: _M_h(__h) { }
template<typename... _Args>
__node_type*
operator()(_Args&&... __args) const
{ return _M_h._M_allocate_node(std::forward<_Args>(__args)...); }
private:
__hashtable_alloc& _M_h;
};
template<bool _Cache_hash_code, bool _Constant_iterators, bool _Unique_keys>
struct _Hashtable_traits
{
using __hash_cached = __bool_constant<_Cache_hash_code>;
using __constant_iterators = __bool_constant<_Constant_iterators>;
using __unique_keys = __bool_constant<_Unique_keys>;
};
template<typename _Hash>
struct _Hashtable_hash_traits
{
static constexpr std::size_t
__small_size_threshold() noexcept
{ return std::__is_fast_hash<_Hash>::value ? 0 : 20; }
};
struct _Hash_node_base
{
_Hash_node_base* _M_nxt;
_Hash_node_base() noexcept : _M_nxt() { }
_Hash_node_base(_Hash_node_base* __next) noexcept : _M_nxt(__next) { }
};
template<typename _Value>
struct _Hash_node_value_base
{
typedef _Value value_type;
__gnu_cxx::__aligned_buffer<_Value> _M_storage;
_Value*
_M_valptr() noexcept
{ return _M_storage._M_ptr(); }
const _Value*
_M_valptr() const noexcept
{ return _M_storage._M_ptr(); }
_Value&
_M_v() noexcept
{ return *_M_valptr(); }
const _Value&
_M_v() const noexcept
{ return *_M_valptr(); }
};
template<bool _Cache_hash_code>
struct _Hash_node_code_cache
{ };
template<>
struct _Hash_node_code_cache<true>
{ std::size_t _M_hash_code; };
template<typename _Value, bool _Cache_hash_code>
struct _Hash_node_value
: _Hash_node_value_base<_Value>
, _Hash_node_code_cache<_Cache_hash_code>
{ };
template<typename _Value, bool _Cache_hash_code>
struct _Hash_node
: _Hash_node_base
, _Hash_node_value<_Value, _Cache_hash_code>
{
_Hash_node*
_M_next() const noexcept
{ return static_cast<_Hash_node*>(this->_M_nxt); }
};
template<typename _Value, bool _Cache_hash_code>
struct _Node_iterator_base
{
using __node_type = _Hash_node<_Value, _Cache_hash_code>;
__node_type* _M_cur;
_Node_iterator_base() : _M_cur(nullptr) { }
_Node_iterator_base(__node_type* __p) noexcept
: _M_cur(__p) { }
void
_M_incr() noexcept
{ _M_cur = _M_cur->_M_next(); }
friend bool
operator==(const _Node_iterator_base& __x, const _Node_iterator_base& __y)
noexcept
{ return __x._M_cur == __y._M_cur; }
friend bool
operator!=(const _Node_iterator_base& __x, const _Node_iterator_base& __y)
noexcept
{ return __x._M_cur != __y._M_cur; }
};
template<typename _Value, bool __constant_iterators, bool __cache>
struct _Node_iterator
: public _Node_iterator_base<_Value, __cache>
{
private:
using __base_type = _Node_iterator_base<_Value, __cache>;
using __node_type = typename __base_type::__node_type;
public:
using value_type = _Value;
using difference_type = std::ptrdiff_t;
using iterator_category = std::forward_iterator_tag;
using pointer = __conditional_t<__constant_iterators,
const value_type*, value_type*>;
using reference = __conditional_t<__constant_iterators,
const value_type&, value_type&>;
_Node_iterator() = default;
explicit
_Node_iterator(__node_type* __p) noexcept
: __base_type(__p) { }
reference
operator*() const noexcept
{ return this->_M_cur->_M_v(); }
pointer
operator->() const noexcept
{ return this->_M_cur->_M_valptr(); }
_Node_iterator&
operator++() noexcept
{
this->_M_incr();
return *this;
}
_Node_iterator
operator++(int) noexcept
{
_Node_iterator __tmp(*this);
this->_M_incr();
return __tmp;
}
};
template<typename _Value, bool __constant_iterators, bool __cache>
struct _Node_const_iterator
: public _Node_iterator_base<_Value, __cache>
{
private:
using __base_type = _Node_iterator_base<_Value, __cache>;
using __node_type = typename __base_type::__node_type;
public:
typedef _Value value_type;
typedef std::ptrdiff_t difference_type;
typedef std::forward_iterator_tag iterator_category;
typedef const value_type* pointer;
typedef const value_type& reference;
_Node_const_iterator() = default;
explicit
_Node_const_iterator(__node_type* __p) noexcept
: __base_type(__p) { }
_Node_const_iterator(const _Node_iterator<_Value, __constant_iterators,
__cache>& __x) noexcept
: __base_type(__x._M_cur) { }
reference
operator*() const noexcept
{ return this->_M_cur->_M_v(); }
pointer
operator->() const noexcept
{ return this->_M_cur->_M_valptr(); }
_Node_const_iterator&
operator++() noexcept
{
this->_M_incr();
return *this;
}
_Node_const_iterator
operator++(int) noexcept
{
_Node_const_iterator __tmp(*this);
this->_M_incr();
return __tmp;
}
};
struct _Mod_range_hashing
{
typedef std::size_t first_argument_type;
typedef std::size_t second_argument_type;
typedef std::size_t result_type;
result_type
operator()(first_argument_type __num,
second_argument_type __den) const noexcept
{ return __num % __den; }
};
struct _Default_ranged_hash { };
struct _Prime_rehash_policy
{
using __has_load_factor = true_type;
_Prime_rehash_policy(float __z = 1.0) noexcept
: _M_max_load_factor(__z), _M_next_resize(0) { }
float
max_load_factor() const noexcept
{ return _M_max_load_factor; }
std::size_t
_M_next_bkt(std::size_t __n) const;
std::size_t
_M_bkt_for_elements(std::size_t __n) const
{ return __builtin_ceil(__n / (double)_M_max_load_factor); }
std::pair<bool, std::size_t>
_M_need_rehash(std::size_t __n_bkt, std::size_t __n_elt,
std::size_t __n_ins) const;
typedef std::size_t _State;
_State
_M_state() const
{ return _M_next_resize; }
void
_M_reset() noexcept
{ _M_next_resize = 0; }
void
_M_reset(_State __state)
{ _M_next_resize = __state; }
static const std::size_t _S_growth_factor = 2;
float _M_max_load_factor;
mutable std::size_t _M_next_resize;
};
struct _Mask_range_hashing
{
typedef std::size_t first_argument_type;
typedef std::size_t second_argument_type;
typedef std::size_t result_type;
result_type
operator()(first_argument_type __num,
second_argument_type __den) const noexcept
{ return __num & (__den - 1); }
};
inline std::size_t
__clp2(std::size_t __n) noexcept
{
using __gnu_cxx::__int_traits;
if (__n < 2)
return __n;
const unsigned __lz = sizeof(size_t) > sizeof(long)
? __builtin_clzll(__n - 1ull)
: __builtin_clzl(__n - 1ul);
return (size_t(1) << (__int_traits<size_t>::__digits - __lz - 1)) << 1;
}
struct _Power2_rehash_policy
{
using __has_load_factor = true_type;
_Power2_rehash_policy(float __z = 1.0) noexcept
: _M_max_load_factor(__z), _M_next_resize(0) { }
float
max_load_factor() const noexcept
{ return _M_max_load_factor; }
std::size_t
_M_next_bkt(std::size_t __n) noexcept
{
if (__n == 0)
return 1;
const auto __max_width = std::min<size_t>(sizeof(size_t), 8);
const auto __max_bkt = size_t(1) << (__max_width * 8 - 1);
std::size_t __res = __clp2(__n);
if (__res == 0)
__res = __max_bkt;
else if (__res == 1)
__res = 2;
if (__res == __max_bkt)
_M_next_resize = size_t(-1);
else
_M_next_resize
= __builtin_floor(__res * (double)_M_max_load_factor);
return __res;
}
std::size_t
_M_bkt_for_elements(std::size_t __n) const noexcept
{ return __builtin_ceil(__n / (double)_M_max_load_factor); }
std::pair<bool, std::size_t>
_M_need_rehash(std::size_t __n_bkt, std::size_t __n_elt,
std::size_t __n_ins) noexcept
{
if (__n_elt + __n_ins > _M_next_resize)
{
double __min_bkts
= std::max<std::size_t>(__n_elt + __n_ins, _M_next_resize ? 0 : 11)
/ (double)_M_max_load_factor;
if (__min_bkts >= __n_bkt)
return { true,
_M_next_bkt(std::max<std::size_t>(__builtin_floor(__min_bkts) + 1,
__n_bkt * _S_growth_factor)) };
_M_next_resize
= __builtin_floor(__n_bkt * (double)_M_max_load_factor);
return { false, 0 };
}
else
return { false, 0 };
}
typedef std::size_t _State;
_State
_M_state() const noexcept
{ return _M_next_resize; }
void
_M_reset() noexcept
{ _M_next_resize = 0; }
void
_M_reset(_State __state) noexcept
{ _M_next_resize = __state; }
static const std::size_t _S_growth_factor = 2;
float _M_max_load_factor;
std::size_t _M_next_resize;
};
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits,
bool _Unique_keys = _Traits::__unique_keys::value>
struct _Map_base { };
template<typename _Key, typename _Val, typename _Alloc, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
struct _Map_base<_Key, pair<const _Key, _Val>, _Alloc, _Select1st, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits, false>
{
using mapped_type = _Val;
};
template<typename _Key, typename _Val, typename _Alloc, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
struct _Map_base<_Key, pair<const _Key, _Val>, _Alloc, _Select1st, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits, true>
{
private:
using __hashtable_base = _Hashtable_base<_Key, pair<const _Key, _Val>,
_Select1st, _Equal, _Hash,
_RangeHash, _Unused,
_Traits>;
using __hashtable = _Hashtable<_Key, pair<const _Key, _Val>, _Alloc,
_Select1st, _Equal, _Hash, _RangeHash,
_Unused, _RehashPolicy, _Traits>;
using __hash_code = typename __hashtable_base::__hash_code;
public:
using key_type = typename __hashtable_base::key_type;
using mapped_type = _Val;
mapped_type&
operator[](const key_type& __k);
mapped_type&
operator[](key_type&& __k);
mapped_type&
at(const key_type& __k)
{
auto __ite = static_cast<__hashtable*>(this)->find(__k);
if (!__ite._M_cur)
__throw_out_of_range(("unordered_map::at"));
return __ite->second;
}
const mapped_type&
at(const key_type& __k) const
{
auto __ite = static_cast<const __hashtable*>(this)->find(__k);
if (!__ite._M_cur)
__throw_out_of_range(("unordered_map::at"));
return __ite->second;
}
};
template<typename _Key, typename _Val, typename _Alloc, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Map_base<_Key, pair<const _Key, _Val>, _Alloc, _Select1st, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits, true>::
operator[](const key_type& __k)
-> mapped_type&
{
__hashtable* __h = static_cast<__hashtable*>(this);
__hash_code __code = __h->_M_hash_code(__k);
std::size_t __bkt = __h->_M_bucket_index(__code);
if (auto __node = __h->_M_find_node(__bkt, __k, __code))
return __node->_M_v().second;
typename __hashtable::_Scoped_node __node {
__h,
std::piecewise_construct,
std::tuple<const key_type&>(__k),
std::tuple<>()
};
auto __pos
= __h->_M_insert_unique_node(__bkt, __code, __node._M_node);
__node._M_node = nullptr;
return __pos->second;
}
template<typename _Key, typename _Val, typename _Alloc, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Map_base<_Key, pair<const _Key, _Val>, _Alloc, _Select1st, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits, true>::
operator[](key_type&& __k)
-> mapped_type&
{
__hashtable* __h = static_cast<__hashtable*>(this);
__hash_code __code = __h->_M_hash_code(__k);
std::size_t __bkt = __h->_M_bucket_index(__code);
if (auto __node = __h->_M_find_node(__bkt, __k, __code))
return __node->_M_v().second;
typename __hashtable::_Scoped_node __node {
__h,
std::piecewise_construct,
std::forward_as_tuple(std::move(__k)),
std::tuple<>()
};
auto __pos
= __h->_M_insert_unique_node(__bkt, __code, __node._M_node);
__node._M_node = nullptr;
return __pos->second;
}
template<typename _Key, typename _Val, typename _Alloc, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits, bool __uniq>
struct _Map_base<const _Key, pair<const _Key, _Val>,
_Alloc, _Select1st, _Equal, _Hash,
_RangeHash, _Unused, _RehashPolicy, _Traits, __uniq>
: _Map_base<_Key, pair<const _Key, _Val>, _Alloc, _Select1st, _Equal, _Hash,
_RangeHash, _Unused, _RehashPolicy, _Traits, __uniq>
{ };
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
struct _Insert_base
{
protected:
using __hashtable_base = _Hashtable_base<_Key, _Value, _ExtractKey,
_Equal, _Hash, _RangeHash,
_Unused, _Traits>;
using __hashtable = _Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash,
_Unused, _RehashPolicy, _Traits>;
using __hash_cached = typename _Traits::__hash_cached;
using __constant_iterators = typename _Traits::__constant_iterators;
using __hashtable_alloc = _Hashtable_alloc<
__alloc_rebind<_Alloc, _Hash_node<_Value,
__hash_cached::value>>>;
using value_type = typename __hashtable_base::value_type;
using size_type = typename __hashtable_base::size_type;
using __unique_keys = typename _Traits::__unique_keys;
using __node_alloc_type = typename __hashtable_alloc::__node_alloc_type;
using __node_gen_type = _AllocNode<__node_alloc_type>;
__hashtable&
_M_conjure_hashtable()
{ return *(static_cast<__hashtable*>(this)); }
template<typename _InputIterator, typename _NodeGetter>
void
_M_insert_range(_InputIterator __first, _InputIterator __last,
const _NodeGetter&, true_type __uks);
template<typename _InputIterator, typename _NodeGetter>
void
_M_insert_range(_InputIterator __first, _InputIterator __last,
const _NodeGetter&, false_type __uks);
public:
using iterator = _Node_iterator<_Value, __constant_iterators::value,
__hash_cached::value>;
using const_iterator = _Node_const_iterator<_Value,
__constant_iterators::value,
__hash_cached::value>;
using __ireturn_type = __conditional_t<__unique_keys::value,
std::pair<iterator, bool>,
iterator>;
__ireturn_type
insert(const value_type& __v)
{
__hashtable& __h = _M_conjure_hashtable();
__node_gen_type __node_gen(__h);
return __h._M_insert(__v, __node_gen, __unique_keys{});
}
iterator
insert(const_iterator __hint, const value_type& __v)
{
__hashtable& __h = _M_conjure_hashtable();
__node_gen_type __node_gen(__h);
return __h._M_insert(__hint, __v, __node_gen, __unique_keys{});
}
template<typename _KType, typename... _Args>
std::pair<iterator, bool>
try_emplace(const_iterator, _KType&& __k, _Args&&... __args)
{
__hashtable& __h = _M_conjure_hashtable();
auto __code = __h._M_hash_code(__k);
std::size_t __bkt = __h._M_bucket_index(__code);
if (auto __node = __h._M_find_node(__bkt, __k, __code))
return { iterator(__node), false };
typename __hashtable::_Scoped_node __node {
&__h,
std::piecewise_construct,
std::forward_as_tuple(std::forward<_KType>(__k)),
std::forward_as_tuple(std::forward<_Args>(__args)...)
};
auto __it
= __h._M_insert_unique_node(__bkt, __code, __node._M_node);
__node._M_node = nullptr;
return { __it, true };
}
void
insert(initializer_list<value_type> __l)
{ this->insert(__l.begin(), __l.end()); }
template<typename _InputIterator>
void
insert(_InputIterator __first, _InputIterator __last)
{
__hashtable& __h = _M_conjure_hashtable();
__node_gen_type __node_gen(__h);
return _M_insert_range(__first, __last, __node_gen, __unique_keys{});
}
};
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
template<typename _InputIterator, typename _NodeGetter>
void
_Insert_base<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused,
_RehashPolicy, _Traits>::
_M_insert_range(_InputIterator __first, _InputIterator __last,
const _NodeGetter& __node_gen, true_type __uks)
{
__hashtable& __h = _M_conjure_hashtable();
for (; __first != __last; ++__first)
__h._M_insert(*__first, __node_gen, __uks);
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
template<typename _InputIterator, typename _NodeGetter>
void
_Insert_base<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused,
_RehashPolicy, _Traits>::
_M_insert_range(_InputIterator __first, _InputIterator __last,
const _NodeGetter& __node_gen, false_type __uks)
{
using __rehash_type = typename __hashtable::__rehash_type;
using __rehash_state = typename __hashtable::__rehash_state;
using pair_type = std::pair<bool, std::size_t>;
size_type __n_elt = __detail::__distance_fw(__first, __last);
if (__n_elt == 0)
return;
__hashtable& __h = _M_conjure_hashtable();
__rehash_type& __rehash = __h._M_rehash_policy;
const __rehash_state& __saved_state = __rehash._M_state();
pair_type __do_rehash = __rehash._M_need_rehash(__h._M_bucket_count,
__h._M_element_count,
__n_elt);
if (__do_rehash.first)
__h._M_rehash(__do_rehash.second, __saved_state);
for (; __first != __last; ++__first)
__h._M_insert(*__first, __node_gen, __uks);
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits,
bool _Constant_iterators = _Traits::__constant_iterators::value>
struct _Insert;
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
struct _Insert<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused,
_RehashPolicy, _Traits, true>
: public _Insert_base<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>
{
using __base_type = _Insert_base<_Key, _Value, _Alloc, _ExtractKey,
_Equal, _Hash, _RangeHash, _Unused,
_RehashPolicy, _Traits>;
using value_type = typename __base_type::value_type;
using iterator = typename __base_type::iterator;
using const_iterator = typename __base_type::const_iterator;
using __ireturn_type = typename __base_type::__ireturn_type;
using __unique_keys = typename __base_type::__unique_keys;
using __hashtable = typename __base_type::__hashtable;
using __node_gen_type = typename __base_type::__node_gen_type;
using __base_type::insert;
__ireturn_type
insert(value_type&& __v)
{
__hashtable& __h = this->_M_conjure_hashtable();
__node_gen_type __node_gen(__h);
return __h._M_insert(std::move(__v), __node_gen, __unique_keys{});
}
iterator
insert(const_iterator __hint, value_type&& __v)
{
__hashtable& __h = this->_M_conjure_hashtable();
__node_gen_type __node_gen(__h);
return __h._M_insert(__hint, std::move(__v), __node_gen,
__unique_keys{});
}
};
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
struct _Insert<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits, false>
: public _Insert_base<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>
{
using __base_type = _Insert_base<_Key, _Value, _Alloc, _ExtractKey,
_Equal, _Hash, _RangeHash, _Unused,
_RehashPolicy, _Traits>;
using value_type = typename __base_type::value_type;
using iterator = typename __base_type::iterator;
using const_iterator = typename __base_type::const_iterator;
using __unique_keys = typename __base_type::__unique_keys;
using __hashtable = typename __base_type::__hashtable;
using __ireturn_type = typename __base_type::__ireturn_type;
using __base_type::insert;
template<typename _Pair>
using __is_cons = std::is_constructible<value_type, _Pair&&>;
template<typename _Pair>
using _IFcons = std::enable_if<__is_cons<_Pair>::value>;
template<typename _Pair>
using _IFconsp = typename _IFcons<_Pair>::type;
template<typename _Pair, typename = _IFconsp<_Pair>>
__ireturn_type
insert(_Pair&& __v)
{
__hashtable& __h = this->_M_conjure_hashtable();
return __h._M_emplace(__unique_keys{}, std::forward<_Pair>(__v));
}
template<typename _Pair, typename = _IFconsp<_Pair>>
iterator
insert(const_iterator __hint, _Pair&& __v)
{
__hashtable& __h = this->_M_conjure_hashtable();
return __h._M_emplace(__hint, __unique_keys{},
std::forward<_Pair>(__v));
}
};
template<typename _Policy>
using __has_load_factor = typename _Policy::__has_load_factor;
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits,
typename =
__detected_or_t<false_type, __has_load_factor, _RehashPolicy>>
struct _Rehash_base;
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
struct _Rehash_base<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits,
false_type >
{
};
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
struct _Rehash_base<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits,
true_type >
{
private:
using __hashtable = _Hashtable<_Key, _Value, _Alloc, _ExtractKey,
_Equal, _Hash, _RangeHash, _Unused,
_RehashPolicy, _Traits>;
public:
float
max_load_factor() const noexcept
{
const __hashtable* __this = static_cast<const __hashtable*>(this);
return __this->__rehash_policy().max_load_factor();
}
void
max_load_factor(float __z)
{
__hashtable* __this = static_cast<__hashtable*>(this);
__this->__rehash_policy(_RehashPolicy(__z));
}
void
reserve(std::size_t __n)
{
__hashtable* __this = static_cast<__hashtable*>(this);
__this->rehash(__this->__rehash_policy()._M_bkt_for_elements(__n));
}
};
template<int _Nm, typename _Tp,
bool __use_ebo = !__is_final(_Tp) && __is_empty(_Tp)>
struct _Hashtable_ebo_helper;
template<int _Nm, typename _Tp>
struct _Hashtable_ebo_helper<_Nm, _Tp, true>
: private _Tp
{
_Hashtable_ebo_helper() noexcept(noexcept(_Tp())) : _Tp() { }
template<typename _OtherTp>
_Hashtable_ebo_helper(_OtherTp&& __tp)
: _Tp(std::forward<_OtherTp>(__tp))
{ }
const _Tp& _M_cget() const { return static_cast<const _Tp&>(*this); }
_Tp& _M_get() { return static_cast<_Tp&>(*this); }
};
template<int _Nm, typename _Tp>
struct _Hashtable_ebo_helper<_Nm, _Tp, false>
{
_Hashtable_ebo_helper() = default;
template<typename _OtherTp>
_Hashtable_ebo_helper(_OtherTp&& __tp)
: _M_tp(std::forward<_OtherTp>(__tp))
{ }
const _Tp& _M_cget() const { return _M_tp; }
_Tp& _M_get() { return _M_tp; }
private:
_Tp _M_tp{};
};
template<typename _Key, typename _Value, typename _ExtractKey,
typename _Hash, typename _RangeHash, typename _Unused,
bool __cache_hash_code>
struct _Local_iterator_base;
template<typename _Key, typename _Value, typename _ExtractKey,
typename _Hash, typename _RangeHash, typename _Unused,
bool __cache_hash_code>
struct _Hash_code_base
: private _Hashtable_ebo_helper<1, _Hash>
{
private:
using __ebo_hash = _Hashtable_ebo_helper<1, _Hash>;
friend struct _Local_iterator_base<_Key, _Value, _ExtractKey,
_Hash, _RangeHash, _Unused, false>;
public:
typedef _Hash hasher;
hasher
hash_function() const
{ return _M_hash(); }
protected:
typedef std::size_t __hash_code;
_Hash_code_base() = default;
_Hash_code_base(const _Hash& __hash) : __ebo_hash(__hash) { }
__hash_code
_M_hash_code(const _Key& __k) const
{
static_assert(__is_invocable<const _Hash&, const _Key&>{},
"hash function must be invocable with an argument of key type");
return _M_hash()(__k);
}
template<typename _Kt>
__hash_code
_M_hash_code_tr(const _Kt& __k) const
{
static_assert(__is_invocable<const _Hash&, const _Kt&>{},
"hash function must be invocable with an argument of key type");
return _M_hash()(__k);
}
__hash_code
_M_hash_code(const _Hash&,
const _Hash_node_value<_Value, true>& __n) const
{ return __n._M_hash_code; }
template<typename _H2>
__hash_code
_M_hash_code(const _H2&,
const _Hash_node_value<_Value, __cache_hash_code>& __n) const
{ return _M_hash_code(_ExtractKey{}(__n._M_v())); }
__hash_code
_M_hash_code(const _Hash_node_value<_Value, false>& __n) const
{ return _M_hash_code(_ExtractKey{}(__n._M_v())); }
__hash_code
_M_hash_code(const _Hash_node_value<_Value, true>& __n) const
{ return __n._M_hash_code; }
std::size_t
_M_bucket_index(__hash_code __c, std::size_t __bkt_count) const
{ return _RangeHash{}(__c, __bkt_count); }
std::size_t
_M_bucket_index(const _Hash_node_value<_Value, false>& __n,
std::size_t __bkt_count) const
noexcept( noexcept(declval<const _Hash&>()(declval<const _Key&>()))
&& noexcept(declval<const _RangeHash&>()((__hash_code)0,
(std::size_t)0)) )
{
return _RangeHash{}(_M_hash_code(_ExtractKey{}(__n._M_v())),
__bkt_count);
}
std::size_t
_M_bucket_index(const _Hash_node_value<_Value, true>& __n,
std::size_t __bkt_count) const
noexcept( noexcept(declval<const _RangeHash&>()((__hash_code)0,
(std::size_t)0)) )
{ return _RangeHash{}(__n._M_hash_code, __bkt_count); }
void
_M_store_code(_Hash_node_code_cache<false>&, __hash_code) const
{ }
void
_M_copy_code(_Hash_node_code_cache<false>&,
const _Hash_node_code_cache<false>&) const
{ }
void
_M_store_code(_Hash_node_code_cache<true>& __n, __hash_code __c) const
{ __n._M_hash_code = __c; }
void
_M_copy_code(_Hash_node_code_cache<true>& __to,
const _Hash_node_code_cache<true>& __from) const
{ __to._M_hash_code = __from._M_hash_code; }
void
_M_swap(_Hash_code_base& __x)
{ std::swap(__ebo_hash::_M_get(), __x.__ebo_hash::_M_get()); }
const _Hash&
_M_hash() const { return __ebo_hash::_M_cget(); }
};
template<typename _Key, typename _Value, typename _ExtractKey,
typename _Hash, typename _RangeHash, typename _Unused>
struct _Local_iterator_base<_Key, _Value, _ExtractKey,
_Hash, _RangeHash, _Unused, true>
: public _Node_iterator_base<_Value, true>
{
protected:
using __base_node_iter = _Node_iterator_base<_Value, true>;
using __hash_code_base = _Hash_code_base<_Key, _Value, _ExtractKey,
_Hash, _RangeHash, _Unused, true>;
_Local_iterator_base() = default;
_Local_iterator_base(const __hash_code_base&,
_Hash_node<_Value, true>* __p,
std::size_t __bkt, std::size_t __bkt_count)
: __base_node_iter(__p), _M_bucket(__bkt), _M_bucket_count(__bkt_count)
{ }
void
_M_incr()
{
__base_node_iter::_M_incr();
if (this->_M_cur)
{
std::size_t __bkt
= _RangeHash{}(this->_M_cur->_M_hash_code, _M_bucket_count);
if (__bkt != _M_bucket)
this->_M_cur = nullptr;
}
}
std::size_t _M_bucket;
std::size_t _M_bucket_count;
public:
std::size_t
_M_get_bucket() const { return _M_bucket; }
};
template<typename _Tp, bool _IsEmpty = std::is_empty<_Tp>::value>
struct _Hash_code_storage
{
__gnu_cxx::__aligned_buffer<_Tp> _M_storage;
_Tp*
_M_h() { return _M_storage._M_ptr(); }
const _Tp*
_M_h() const { return _M_storage._M_ptr(); }
};
template<typename _Tp>
struct _Hash_code_storage<_Tp, true>
{
static_assert( std::is_empty<_Tp>::value, "Type must be empty" );
_Tp*
_M_h() { return reinterpret_cast<_Tp*>(this); }
const _Tp*
_M_h() const { return reinterpret_cast<const _Tp*>(this); }
};
template<typename _Key, typename _Value, typename _ExtractKey,
typename _Hash, typename _RangeHash, typename _Unused>
using __hash_code_for_local_iter
= _Hash_code_storage<_Hash_code_base<_Key, _Value, _ExtractKey,
_Hash, _RangeHash, _Unused, false>>;
template<typename _Key, typename _Value, typename _ExtractKey,
typename _Hash, typename _RangeHash, typename _Unused>
struct _Local_iterator_base<_Key, _Value, _ExtractKey,
_Hash, _RangeHash, _Unused, false>
: __hash_code_for_local_iter<_Key, _Value, _ExtractKey, _Hash, _RangeHash,
_Unused>
, _Node_iterator_base<_Value, false>
{
protected:
using __hash_code_base = _Hash_code_base<_Key, _Value, _ExtractKey,
_Hash, _RangeHash, _Unused, false>;
using __node_iter_base = _Node_iterator_base<_Value, false>;
_Local_iterator_base() : _M_bucket_count(-1) { }
_Local_iterator_base(const __hash_code_base& __base,
_Hash_node<_Value, false>* __p,
std::size_t __bkt, std::size_t __bkt_count)
: __node_iter_base(__p), _M_bucket(__bkt), _M_bucket_count(__bkt_count)
{ _M_init(__base); }
~_Local_iterator_base()
{
if (_M_bucket_count != size_t(-1))
_M_destroy();
}
_Local_iterator_base(const _Local_iterator_base& __iter)
: __node_iter_base(__iter._M_cur), _M_bucket(__iter._M_bucket)
, _M_bucket_count(__iter._M_bucket_count)
{
if (_M_bucket_count != size_t(-1))
_M_init(*__iter._M_h());
}
_Local_iterator_base&
operator=(const _Local_iterator_base& __iter)
{
if (_M_bucket_count != -1)
_M_destroy();
this->_M_cur = __iter._M_cur;
_M_bucket = __iter._M_bucket;
_M_bucket_count = __iter._M_bucket_count;
if (_M_bucket_count != -1)
_M_init(*__iter._M_h());
return *this;
}
void
_M_incr()
{
__node_iter_base::_M_incr();
if (this->_M_cur)
{
std::size_t __bkt = this->_M_h()->_M_bucket_index(*this->_M_cur,
_M_bucket_count);
if (__bkt != _M_bucket)
this->_M_cur = nullptr;
}
}
std::size_t _M_bucket;
std::size_t _M_bucket_count;
void
_M_init(const __hash_code_base& __base)
{ ::new(this->_M_h()) __hash_code_base(__base); }
void
_M_destroy() { this->_M_h()->~__hash_code_base(); }
public:
std::size_t
_M_get_bucket() const { return _M_bucket; }
};
template<typename _Key, typename _Value, typename _ExtractKey,
typename _Hash, typename _RangeHash, typename _Unused,
bool __constant_iterators, bool __cache>
struct _Local_iterator
: public _Local_iterator_base<_Key, _Value, _ExtractKey,
_Hash, _RangeHash, _Unused, __cache>
{
private:
using __base_type = _Local_iterator_base<_Key, _Value, _ExtractKey,
_Hash, _RangeHash, _Unused, __cache>;
using __hash_code_base = typename __base_type::__hash_code_base;
public:
using value_type = _Value;
using pointer = __conditional_t<__constant_iterators,
const value_type*, value_type*>;
using reference = __conditional_t<__constant_iterators,
const value_type&, value_type&>;
using difference_type = ptrdiff_t;
using iterator_category = forward_iterator_tag;
_Local_iterator() = default;
_Local_iterator(const __hash_code_base& __base,
_Hash_node<_Value, __cache>* __n,
std::size_t __bkt, std::size_t __bkt_count)
: __base_type(__base, __n, __bkt, __bkt_count)
{ }
reference
operator*() const
{ return this->_M_cur->_M_v(); }
pointer
operator->() const
{ return this->_M_cur->_M_valptr(); }
_Local_iterator&
operator++()
{
this->_M_incr();
return *this;
}
_Local_iterator
operator++(int)
{
_Local_iterator __tmp(*this);
this->_M_incr();
return __tmp;
}
};
template<typename _Key, typename _Value, typename _ExtractKey,
typename _Hash, typename _RangeHash, typename _Unused,
bool __constant_iterators, bool __cache>
struct _Local_const_iterator
: public _Local_iterator_base<_Key, _Value, _ExtractKey,
_Hash, _RangeHash, _Unused, __cache>
{
private:
using __base_type = _Local_iterator_base<_Key, _Value, _ExtractKey,
_Hash, _RangeHash, _Unused, __cache>;
using __hash_code_base = typename __base_type::__hash_code_base;
public:
typedef _Value value_type;
typedef const value_type* pointer;
typedef const value_type& reference;
typedef std::ptrdiff_t difference_type;
typedef std::forward_iterator_tag iterator_category;
_Local_const_iterator() = default;
_Local_const_iterator(const __hash_code_base& __base,
_Hash_node<_Value, __cache>* __n,
std::size_t __bkt, std::size_t __bkt_count)
: __base_type(__base, __n, __bkt, __bkt_count)
{ }
_Local_const_iterator(const _Local_iterator<_Key, _Value, _ExtractKey,
_Hash, _RangeHash, _Unused,
__constant_iterators,
__cache>& __x)
: __base_type(__x)
{ }
reference
operator*() const
{ return this->_M_cur->_M_v(); }
pointer
operator->() const
{ return this->_M_cur->_M_valptr(); }
_Local_const_iterator&
operator++()
{
this->_M_incr();
return *this;
}
_Local_const_iterator
operator++(int)
{
_Local_const_iterator __tmp(*this);
this->_M_incr();
return __tmp;
}
};
template<typename _Key, typename _Value, typename _ExtractKey,
typename _Equal, typename _Hash, typename _RangeHash,
typename _Unused, typename _Traits>
struct _Hashtable_base
: public _Hash_code_base<_Key, _Value, _ExtractKey, _Hash, _RangeHash,
_Unused, _Traits::__hash_cached::value>,
private _Hashtable_ebo_helper<0, _Equal>
{
public:
typedef _Key key_type;
typedef _Value value_type;
typedef _Equal key_equal;
typedef std::size_t size_type;
typedef std::ptrdiff_t difference_type;
using __traits_type = _Traits;
using __hash_cached = typename __traits_type::__hash_cached;
using __hash_code_base = _Hash_code_base<_Key, _Value, _ExtractKey,
_Hash, _RangeHash, _Unused,
__hash_cached::value>;
using __hash_code = typename __hash_code_base::__hash_code;
private:
using _EqualEBO = _Hashtable_ebo_helper<0, _Equal>;
static bool
_S_equals(__hash_code, const _Hash_node_code_cache<false>&)
{ return true; }
static bool
_S_node_equals(const _Hash_node_code_cache<false>&,
const _Hash_node_code_cache<false>&)
{ return true; }
static bool
_S_equals(__hash_code __c, const _Hash_node_code_cache<true>& __n)
{ return __c == __n._M_hash_code; }
static bool
_S_node_equals(const _Hash_node_code_cache<true>& __lhn,
const _Hash_node_code_cache<true>& __rhn)
{ return __lhn._M_hash_code == __rhn._M_hash_code; }
protected:
_Hashtable_base() = default;
_Hashtable_base(const _Hash& __hash, const _Equal& __eq)
: __hash_code_base(__hash), _EqualEBO(__eq)
{ }
bool
_M_key_equals(const _Key& __k,
const _Hash_node_value<_Value,
__hash_cached::value>& __n) const
{
static_assert(__is_invocable<const _Equal&, const _Key&, const _Key&>{},
"key equality predicate must be invocable with two arguments of "
"key type");
return _M_eq()(__k, _ExtractKey{}(__n._M_v()));
}
template<typename _Kt>
bool
_M_key_equals_tr(const _Kt& __k,
const _Hash_node_value<_Value,
__hash_cached::value>& __n) const
{
static_assert(
__is_invocable<const _Equal&, const _Kt&, const _Key&>{},
"key equality predicate must be invocable with two arguments of "
"key type");
return _M_eq()(__k, _ExtractKey{}(__n._M_v()));
}
bool
_M_equals(const _Key& __k, __hash_code __c,
const _Hash_node_value<_Value, __hash_cached::value>& __n) const
{ return _S_equals(__c, __n) && _M_key_equals(__k, __n); }
template<typename _Kt>
bool
_M_equals_tr(const _Kt& __k, __hash_code __c,
const _Hash_node_value<_Value,
__hash_cached::value>& __n) const
{ return _S_equals(__c, __n) && _M_key_equals_tr(__k, __n); }
bool
_M_node_equals(
const _Hash_node_value<_Value, __hash_cached::value>& __lhn,
const _Hash_node_value<_Value, __hash_cached::value>& __rhn) const
{
return _S_node_equals(__lhn, __rhn)
&& _M_key_equals(_ExtractKey{}(__lhn._M_v()), __rhn);
}
void
_M_swap(_Hashtable_base& __x)
{
__hash_code_base::_M_swap(__x);
std::swap(_EqualEBO::_M_get(), __x._EqualEBO::_M_get());
}
const _Equal&
_M_eq() const { return _EqualEBO::_M_cget(); }
};
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits,
bool _Unique_keys = _Traits::__unique_keys::value>
struct _Equality;
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
struct _Equality<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits, true>
{
using __hashtable = _Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused,
_RehashPolicy, _Traits>;
bool
_M_equal(const __hashtable&) const;
};
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
bool
_Equality<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits, true>::
_M_equal(const __hashtable& __other) const
{
using __node_type = typename __hashtable::__node_type;
const __hashtable* __this = static_cast<const __hashtable*>(this);
if (__this->size() != __other.size())
return false;
for (auto __itx = __this->begin(); __itx != __this->end(); ++__itx)
{
std::size_t __ybkt = __other._M_bucket_index(*__itx._M_cur);
auto __prev_n = __other._M_buckets[__ybkt];
if (!__prev_n)
return false;
for (__node_type* __n = static_cast<__node_type*>(__prev_n->_M_nxt);;
__n = __n->_M_next())
{
if (__n->_M_v() == *__itx)
break;
if (!__n->_M_nxt
|| __other._M_bucket_index(*__n->_M_next()) != __ybkt)
return false;
}
}
return true;
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
struct _Equality<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits, false>
{
using __hashtable = _Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused,
_RehashPolicy, _Traits>;
bool
_M_equal(const __hashtable&) const;
};
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
bool
_Equality<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits, false>::
_M_equal(const __hashtable& __other) const
{
using __node_type = typename __hashtable::__node_type;
const __hashtable* __this = static_cast<const __hashtable*>(this);
if (__this->size() != __other.size())
return false;
for (auto __itx = __this->begin(); __itx != __this->end();)
{
std::size_t __x_count = 1;
auto __itx_end = __itx;
for (++__itx_end; __itx_end != __this->end()
&& __this->key_eq()(_ExtractKey{}(*__itx),
_ExtractKey{}(*__itx_end));
++__itx_end)
++__x_count;
std::size_t __ybkt = __other._M_bucket_index(*__itx._M_cur);
auto __y_prev_n = __other._M_buckets[__ybkt];
if (!__y_prev_n)
return false;
__node_type* __y_n = static_cast<__node_type*>(__y_prev_n->_M_nxt);
for (;;)
{
if (__this->key_eq()(_ExtractKey{}(__y_n->_M_v()),
_ExtractKey{}(*__itx)))
break;
auto __y_ref_n = __y_n;
for (__y_n = __y_n->_M_next(); __y_n; __y_n = __y_n->_M_next())
if (!__other._M_node_equals(*__y_ref_n, *__y_n))
break;
if (!__y_n || __other._M_bucket_index(*__y_n) != __ybkt)
return false;
}
typename __hashtable::const_iterator __ity(__y_n);
for (auto __ity_end = __ity; __ity_end != __other.end(); ++__ity_end)
if (--__x_count == 0)
break;
if (__x_count != 0)
return false;
if (!std::is_permutation(__itx, __itx_end, __ity))
return false;
__itx = __itx_end;
}
return true;
}
template<typename _NodeAlloc>
struct _Hashtable_alloc : private _Hashtable_ebo_helper<0, _NodeAlloc>
{
private:
using __ebo_node_alloc = _Hashtable_ebo_helper<0, _NodeAlloc>;
template<typename>
struct __get_value_type;
template<typename _Val, bool _Cache_hash_code>
struct __get_value_type<_Hash_node<_Val, _Cache_hash_code>>
{ using type = _Val; };
public:
using __node_type = typename _NodeAlloc::value_type;
using __node_alloc_type = _NodeAlloc;
using __node_alloc_traits = __gnu_cxx::__alloc_traits<__node_alloc_type>;
using __value_alloc_traits = typename __node_alloc_traits::template
rebind_traits<typename __get_value_type<__node_type>::type>;
using __node_ptr = __node_type*;
using __node_base = _Hash_node_base;
using __node_base_ptr = __node_base*;
using __buckets_alloc_type =
__alloc_rebind<__node_alloc_type, __node_base_ptr>;
using __buckets_alloc_traits = std::allocator_traits<__buckets_alloc_type>;
using __buckets_ptr = __node_base_ptr*;
_Hashtable_alloc() = default;
_Hashtable_alloc(const _Hashtable_alloc&) = default;
_Hashtable_alloc(_Hashtable_alloc&&) = default;
template<typename _Alloc>
_Hashtable_alloc(_Alloc&& __a)
: __ebo_node_alloc(std::forward<_Alloc>(__a))
{ }
__node_alloc_type&
_M_node_allocator()
{ return __ebo_node_alloc::_M_get(); }
const __node_alloc_type&
_M_node_allocator() const
{ return __ebo_node_alloc::_M_cget(); }
template<typename... _Args>
__node_ptr
_M_allocate_node(_Args&&... __args);
void
_M_deallocate_node(__node_ptr __n);
void
_M_deallocate_node_ptr(__node_ptr __n);
void
_M_deallocate_nodes(__node_ptr __n);
__buckets_ptr
_M_allocate_buckets(std::size_t __bkt_count);
void
_M_deallocate_buckets(__buckets_ptr, std::size_t __bkt_count);
};
template<typename _NodeAlloc>
template<typename... _Args>
auto
_Hashtable_alloc<_NodeAlloc>::_M_allocate_node(_Args&&... __args)
-> __node_ptr
{
auto __nptr = __node_alloc_traits::allocate(_M_node_allocator(), 1);
__node_ptr __n = std::__to_address(__nptr);
try
{
::new ((void*)__n) __node_type;
__node_alloc_traits::construct(_M_node_allocator(),
__n->_M_valptr(),
std::forward<_Args>(__args)...);
return __n;
}
catch(...)
{
__node_alloc_traits::deallocate(_M_node_allocator(), __nptr, 1);
throw;
}
}
template<typename _NodeAlloc>
void
_Hashtable_alloc<_NodeAlloc>::_M_deallocate_node(__node_ptr __n)
{
__node_alloc_traits::destroy(_M_node_allocator(), __n->_M_valptr());
_M_deallocate_node_ptr(__n);
}
template<typename _NodeAlloc>
void
_Hashtable_alloc<_NodeAlloc>::_M_deallocate_node_ptr(__node_ptr __n)
{
typedef typename __node_alloc_traits::pointer _Ptr;
auto __ptr = std::pointer_traits<_Ptr>::pointer_to(*__n);
__n->~__node_type();
__node_alloc_traits::deallocate(_M_node_allocator(), __ptr, 1);
}
template<typename _NodeAlloc>
void
_Hashtable_alloc<_NodeAlloc>::_M_deallocate_nodes(__node_ptr __n)
{
while (__n)
{
__node_ptr __tmp = __n;
__n = __n->_M_next();
_M_deallocate_node(__tmp);
}
}
template<typename _NodeAlloc>
auto
_Hashtable_alloc<_NodeAlloc>::_M_allocate_buckets(std::size_t __bkt_count)
-> __buckets_ptr
{
__buckets_alloc_type __alloc(_M_node_allocator());
auto __ptr = __buckets_alloc_traits::allocate(__alloc, __bkt_count);
__buckets_ptr __p = std::__to_address(__ptr);
__builtin_memset(__p, 0, __bkt_count * sizeof(__node_base_ptr));
return __p;
}
template<typename _NodeAlloc>
void
_Hashtable_alloc<_NodeAlloc>::
_M_deallocate_buckets(__buckets_ptr __bkts,
std::size_t __bkt_count)
{
typedef typename __buckets_alloc_traits::pointer _Ptr;
auto __ptr = std::pointer_traits<_Ptr>::pointer_to(*__bkts);
__buckets_alloc_type __alloc(_M_node_allocator());
__buckets_alloc_traits::deallocate(__alloc, __ptr, __bkt_count);
}
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Tp, typename _Hash>
using __cache_default
= __not_<__and_<
__is_fast_hash<_Hash>,
__is_nothrow_invocable<const _Hash&, const _Tp&>>>;
template<typename _Equal, typename _Hash, typename _Allocator>
using _Hashtable_enable_default_ctor
= _Enable_default_constructor<__and_<is_default_constructible<_Equal>,
is_default_constructible<_Hash>,
is_default_constructible<_Allocator>>{},
__detail::_Hash_node_base>;
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
class _Hashtable
: public __detail::_Hashtable_base<_Key, _Value, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _Traits>,
public __detail::_Map_base<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused,
_RehashPolicy, _Traits>,
public __detail::_Insert<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused,
_RehashPolicy, _Traits>,
public __detail::_Rehash_base<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused,
_RehashPolicy, _Traits>,
public __detail::_Equality<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused,
_RehashPolicy, _Traits>,
private __detail::_Hashtable_alloc<
__alloc_rebind<_Alloc,
__detail::_Hash_node<_Value,
_Traits::__hash_cached::value>>>,
private _Hashtable_enable_default_ctor<_Equal, _Hash, _Alloc>
{
static_assert(is_same<typename remove_cv<_Value>::type, _Value>::value,
"unordered container must have a non-const, non-volatile value_type");
using __traits_type = _Traits;
using __hash_cached = typename __traits_type::__hash_cached;
using __constant_iterators = typename __traits_type::__constant_iterators;
using __node_type = __detail::_Hash_node<_Value, __hash_cached::value>;
using __node_alloc_type = __alloc_rebind<_Alloc, __node_type>;
using __hashtable_alloc = __detail::_Hashtable_alloc<__node_alloc_type>;
using __node_value_type =
__detail::_Hash_node_value<_Value, __hash_cached::value>;
using __node_ptr = typename __hashtable_alloc::__node_ptr;
using __value_alloc_traits =
typename __hashtable_alloc::__value_alloc_traits;
using __node_alloc_traits =
typename __hashtable_alloc::__node_alloc_traits;
using __node_base = typename __hashtable_alloc::__node_base;
using __node_base_ptr = typename __hashtable_alloc::__node_base_ptr;
using __buckets_ptr = typename __hashtable_alloc::__buckets_ptr;
using __insert_base = __detail::_Insert<_Key, _Value, _Alloc, _ExtractKey,
_Equal, _Hash,
_RangeHash, _Unused,
_RehashPolicy, _Traits>;
using __enable_default_ctor
= _Hashtable_enable_default_ctor<_Equal, _Hash, _Alloc>;
public:
typedef _Key key_type;
typedef _Value value_type;
typedef _Alloc allocator_type;
typedef _Equal key_equal;
typedef typename __value_alloc_traits::pointer pointer;
typedef typename __value_alloc_traits::const_pointer const_pointer;
typedef value_type& reference;
typedef const value_type& const_reference;
using iterator = typename __insert_base::iterator;
using const_iterator = typename __insert_base::const_iterator;
using local_iterator = __detail::_Local_iterator<key_type, _Value,
_ExtractKey, _Hash, _RangeHash, _Unused,
__constant_iterators::value,
__hash_cached::value>;
using const_local_iterator = __detail::_Local_const_iterator<
key_type, _Value,
_ExtractKey, _Hash, _RangeHash, _Unused,
__constant_iterators::value, __hash_cached::value>;
private:
using __rehash_type = _RehashPolicy;
using __rehash_state = typename __rehash_type::_State;
using __unique_keys = typename __traits_type::__unique_keys;
using __hashtable_base = __detail::
_Hashtable_base<_Key, _Value, _ExtractKey,
_Equal, _Hash, _RangeHash, _Unused, _Traits>;
using __hash_code_base = typename __hashtable_base::__hash_code_base;
using __hash_code = typename __hashtable_base::__hash_code;
using __ireturn_type = typename __insert_base::__ireturn_type;
using __map_base = __detail::_Map_base<_Key, _Value, _Alloc, _ExtractKey,
_Equal, _Hash, _RangeHash, _Unused,
_RehashPolicy, _Traits>;
using __rehash_base = __detail::_Rehash_base<_Key, _Value, _Alloc,
_ExtractKey, _Equal,
_Hash, _RangeHash, _Unused,
_RehashPolicy, _Traits>;
using __eq_base = __detail::_Equality<_Key, _Value, _Alloc, _ExtractKey,
_Equal, _Hash, _RangeHash, _Unused,
_RehashPolicy, _Traits>;
using __reuse_or_alloc_node_gen_t =
__detail::_ReuseOrAllocNode<__node_alloc_type>;
using __alloc_node_gen_t =
__detail::_AllocNode<__node_alloc_type>;
using __node_builder_t =
__detail::_NodeBuilder<_ExtractKey>;
struct _Scoped_node
{
_Scoped_node(__node_ptr __n, __hashtable_alloc* __h)
: _M_h(__h), _M_node(__n) { }
template<typename... _Args>
_Scoped_node(__hashtable_alloc* __h, _Args&&... __args)
: _M_h(__h),
_M_node(__h->_M_allocate_node(std::forward<_Args>(__args)...))
{ }
~_Scoped_node() { if (_M_node) _M_h->_M_deallocate_node(_M_node); };
_Scoped_node(const _Scoped_node&) = delete;
_Scoped_node& operator=(const _Scoped_node&) = delete;
__hashtable_alloc* _M_h;
__node_ptr _M_node;
};
template<typename _Ht>
static constexpr
__conditional_t<std::is_lvalue_reference<_Ht>::value,
const value_type&, value_type&&>
__fwd_value_for(value_type& __val) noexcept
{ return std::move(__val); }
struct __hash_code_base_access : __hash_code_base
{ using __hash_code_base::_M_bucket_index; };
static_assert(is_nothrow_default_constructible<_RangeHash>::value,
"Functor used to map hash code to bucket index"
" must be nothrow default constructible");
static_assert(noexcept(
std::declval<const _RangeHash&>()((std::size_t)0, (std::size_t)0)),
"Functor used to map hash code to bucket index must be"
" noexcept");
static_assert(is_nothrow_default_constructible<_ExtractKey>::value,
"_ExtractKey must be nothrow default constructible");
static_assert(noexcept(
std::declval<const _ExtractKey&>()(std::declval<_Value>())),
"_ExtractKey functor must be noexcept invocable");
template<typename _Keya, typename _Valuea, typename _Alloca,
typename _ExtractKeya, typename _Equala,
typename _Hasha, typename _RangeHasha, typename _Unuseda,
typename _RehashPolicya, typename _Traitsa,
bool _Unique_keysa>
friend struct __detail::_Map_base;
template<typename _Keya, typename _Valuea, typename _Alloca,
typename _ExtractKeya, typename _Equala,
typename _Hasha, typename _RangeHasha, typename _Unuseda,
typename _RehashPolicya, typename _Traitsa>
friend struct __detail::_Insert_base;
template<typename _Keya, typename _Valuea, typename _Alloca,
typename _ExtractKeya, typename _Equala,
typename _Hasha, typename _RangeHasha, typename _Unuseda,
typename _RehashPolicya, typename _Traitsa,
bool _Constant_iteratorsa>
friend struct __detail::_Insert;
template<typename _Keya, typename _Valuea, typename _Alloca,
typename _ExtractKeya, typename _Equala,
typename _Hasha, typename _RangeHasha, typename _Unuseda,
typename _RehashPolicya, typename _Traitsa,
bool _Unique_keysa>
friend struct __detail::_Equality;
public:
using size_type = typename __hashtable_base::size_type;
using difference_type = typename __hashtable_base::difference_type;
using node_type = _Node_handle<_Key, _Value, __node_alloc_type>;
using insert_return_type = _Node_insert_return<iterator, node_type>;
private:
__buckets_ptr _M_buckets = &_M_single_bucket;
size_type _M_bucket_count = 1;
__node_base _M_before_begin;
size_type _M_element_count = 0;
_RehashPolicy _M_rehash_policy;
__node_base_ptr _M_single_bucket = nullptr;
void
_M_update_bbegin()
{
if (_M_begin())
_M_buckets[_M_bucket_index(*_M_begin())] = &_M_before_begin;
}
void
_M_update_bbegin(__node_ptr __n)
{
_M_before_begin._M_nxt = __n;
_M_update_bbegin();
}
bool
_M_uses_single_bucket(__buckets_ptr __bkts) const
{ return __builtin_expect(__bkts == &_M_single_bucket, false); }
bool
_M_uses_single_bucket() const
{ return _M_uses_single_bucket(_M_buckets); }
static constexpr size_t
__small_size_threshold() noexcept
{
return
__detail::_Hashtable_hash_traits<_Hash>::__small_size_threshold();
}
__hashtable_alloc&
_M_base_alloc() { return *this; }
__buckets_ptr
_M_allocate_buckets(size_type __bkt_count)
{
if (__builtin_expect(__bkt_count == 1, false))
{
_M_single_bucket = nullptr;
return &_M_single_bucket;
}
return __hashtable_alloc::_M_allocate_buckets(__bkt_count);
}
void
_M_deallocate_buckets(__buckets_ptr __bkts, size_type __bkt_count)
{
if (_M_uses_single_bucket(__bkts))
return;
__hashtable_alloc::_M_deallocate_buckets(__bkts, __bkt_count);
}
void
_M_deallocate_buckets()
{ _M_deallocate_buckets(_M_buckets, _M_bucket_count); }
__node_ptr
_M_bucket_begin(size_type __bkt) const;
__node_ptr
_M_begin() const
{ return static_cast<__node_ptr>(_M_before_begin._M_nxt); }
template<typename _Ht>
void
_M_assign_elements(_Ht&&);
template<typename _Ht, typename _NodeGenerator>
void
_M_assign(_Ht&&, const _NodeGenerator&);
void
_M_move_assign(_Hashtable&&, true_type);
void
_M_move_assign(_Hashtable&&, false_type);
void
_M_reset() noexcept;
_Hashtable(const _Hash& __h, const _Equal& __eq,
const allocator_type& __a)
: __hashtable_base(__h, __eq),
__hashtable_alloc(__node_alloc_type(__a)),
__enable_default_ctor(_Enable_default_constructor_tag{})
{ }
template<bool _No_realloc = true>
static constexpr bool
_S_nothrow_move()
{
if constexpr (_No_realloc)
if constexpr (is_nothrow_copy_constructible<_Hash>())
return is_nothrow_copy_constructible<_Equal>();
return false;
}
_Hashtable(_Hashtable&& __ht, __node_alloc_type&& __a,
true_type )
noexcept(_S_nothrow_move());
_Hashtable(_Hashtable&&, __node_alloc_type&&,
false_type );
template<typename _InputIterator>
_Hashtable(_InputIterator __first, _InputIterator __last,
size_type __bkt_count_hint,
const _Hash&, const _Equal&, const allocator_type&,
true_type __uks);
template<typename _InputIterator>
_Hashtable(_InputIterator __first, _InputIterator __last,
size_type __bkt_count_hint,
const _Hash&, const _Equal&, const allocator_type&,
false_type __uks);
public:
_Hashtable() = default;
_Hashtable(const _Hashtable&);
_Hashtable(const _Hashtable&, const allocator_type&);
explicit
_Hashtable(size_type __bkt_count_hint,
const _Hash& __hf = _Hash(),
const key_equal& __eql = key_equal(),
const allocator_type& __a = allocator_type());
_Hashtable(_Hashtable&& __ht)
noexcept(_S_nothrow_move())
: _Hashtable(std::move(__ht), std::move(__ht._M_node_allocator()),
true_type{})
{ }
_Hashtable(_Hashtable&& __ht, const allocator_type& __a)
noexcept(_S_nothrow_move<__node_alloc_traits::_S_always_equal()>())
: _Hashtable(std::move(__ht), __node_alloc_type(__a),
typename __node_alloc_traits::is_always_equal{})
{ }
explicit
_Hashtable(const allocator_type& __a)
: __hashtable_alloc(__node_alloc_type(__a)),
__enable_default_ctor(_Enable_default_constructor_tag{})
{ }
template<typename _InputIterator>
_Hashtable(_InputIterator __f, _InputIterator __l,
size_type __bkt_count_hint = 0,
const _Hash& __hf = _Hash(),
const key_equal& __eql = key_equal(),
const allocator_type& __a = allocator_type())
: _Hashtable(__f, __l, __bkt_count_hint, __hf, __eql, __a,
__unique_keys{})
{ }
_Hashtable(initializer_list<value_type> __l,
size_type __bkt_count_hint = 0,
const _Hash& __hf = _Hash(),
const key_equal& __eql = key_equal(),
const allocator_type& __a = allocator_type())
: _Hashtable(__l.begin(), __l.end(), __bkt_count_hint,
__hf, __eql, __a, __unique_keys{})
{ }
_Hashtable&
operator=(const _Hashtable& __ht);
_Hashtable&
operator=(_Hashtable&& __ht)
noexcept(__node_alloc_traits::_S_nothrow_move()
&& is_nothrow_move_assignable<_Hash>::value
&& is_nothrow_move_assignable<_Equal>::value)
{
constexpr bool __move_storage =
__node_alloc_traits::_S_propagate_on_move_assign()
|| __node_alloc_traits::_S_always_equal();
_M_move_assign(std::move(__ht), __bool_constant<__move_storage>());
return *this;
}
_Hashtable&
operator=(initializer_list<value_type> __l)
{
__reuse_or_alloc_node_gen_t __roan(_M_begin(), *this);
_M_before_begin._M_nxt = nullptr;
clear();
auto __l_bkt_count = _M_rehash_policy._M_bkt_for_elements(__l.size());
if (_M_bucket_count < __l_bkt_count)
rehash(__l_bkt_count);
this->_M_insert_range(__l.begin(), __l.end(), __roan, __unique_keys{});
return *this;
}
~_Hashtable() noexcept;
void
swap(_Hashtable&)
noexcept(__and_<__is_nothrow_swappable<_Hash>,
__is_nothrow_swappable<_Equal>>::value);
iterator
begin() noexcept
{ return iterator(_M_begin()); }
const_iterator
begin() const noexcept
{ return const_iterator(_M_begin()); }
iterator
end() noexcept
{ return iterator(nullptr); }
const_iterator
end() const noexcept
{ return const_iterator(nullptr); }
const_iterator
cbegin() const noexcept
{ return const_iterator(_M_begin()); }
const_iterator
cend() const noexcept
{ return const_iterator(nullptr); }
size_type
size() const noexcept
{ return _M_element_count; }
[[__nodiscard__]] bool
empty() const noexcept
{ return size() == 0; }
allocator_type
get_allocator() const noexcept
{ return allocator_type(this->_M_node_allocator()); }
size_type
max_size() const noexcept
{ return __node_alloc_traits::max_size(this->_M_node_allocator()); }
key_equal
key_eq() const
{ return this->_M_eq(); }
size_type
bucket_count() const noexcept
{ return _M_bucket_count; }
size_type
max_bucket_count() const noexcept
{ return max_size(); }
size_type
bucket_size(size_type __bkt) const
{ return std::distance(begin(__bkt), end(__bkt)); }
size_type
bucket(const key_type& __k) const
{ return _M_bucket_index(this->_M_hash_code(__k)); }
local_iterator
begin(size_type __bkt)
{
return local_iterator(*this, _M_bucket_begin(__bkt),
__bkt, _M_bucket_count);
}
local_iterator
end(size_type __bkt)
{ return local_iterator(*this, nullptr, __bkt, _M_bucket_count); }
const_local_iterator
begin(size_type __bkt) const
{
return const_local_iterator(*this, _M_bucket_begin(__bkt),
__bkt, _M_bucket_count);
}
const_local_iterator
end(size_type __bkt) const
{ return const_local_iterator(*this, nullptr, __bkt, _M_bucket_count); }
const_local_iterator
cbegin(size_type __bkt) const
{
return const_local_iterator(*this, _M_bucket_begin(__bkt),
__bkt, _M_bucket_count);
}
const_local_iterator
cend(size_type __bkt) const
{ return const_local_iterator(*this, nullptr, __bkt, _M_bucket_count); }
float
load_factor() const noexcept
{
return static_cast<float>(size()) / static_cast<float>(bucket_count());
}
const _RehashPolicy&
__rehash_policy() const
{ return _M_rehash_policy; }
void
__rehash_policy(const _RehashPolicy& __pol)
{ _M_rehash_policy = __pol; }
iterator
find(const key_type& __k);
const_iterator
find(const key_type& __k) const;
size_type
count(const key_type& __k) const;
std::pair<iterator, iterator>
equal_range(const key_type& __k);
std::pair<const_iterator, const_iterator>
equal_range(const key_type& __k) const;
private:
size_type
_M_bucket_index(const __node_value_type& __n) const noexcept
{ return __hash_code_base::_M_bucket_index(__n, _M_bucket_count); }
size_type
_M_bucket_index(__hash_code __c) const
{ return __hash_code_base::_M_bucket_index(__c, _M_bucket_count); }
__node_base_ptr
_M_find_before_node(const key_type&);
__node_base_ptr
_M_find_before_node(size_type, const key_type&, __hash_code) const;
template<typename _Kt>
__node_base_ptr
_M_find_before_node_tr(size_type, const _Kt&, __hash_code) const;
__node_ptr
_M_find_node(size_type __bkt, const key_type& __key,
__hash_code __c) const
{
__node_base_ptr __before_n = _M_find_before_node(__bkt, __key, __c);
if (__before_n)
return static_cast<__node_ptr>(__before_n->_M_nxt);
return nullptr;
}
template<typename _Kt>
__node_ptr
_M_find_node_tr(size_type __bkt, const _Kt& __key,
__hash_code __c) const
{
auto __before_n = _M_find_before_node_tr(__bkt, __key, __c);
if (__before_n)
return static_cast<__node_ptr>(__before_n->_M_nxt);
return nullptr;
}
void
_M_insert_bucket_begin(size_type, __node_ptr);
void
_M_remove_bucket_begin(size_type __bkt, __node_ptr __next_n,
size_type __next_bkt);
__node_base_ptr
_M_get_previous_node(size_type __bkt, __node_ptr __n);
pair<const_iterator, __hash_code>
_M_compute_hash_code(const_iterator __hint, const key_type& __k) const;
iterator
_M_insert_unique_node(size_type __bkt, __hash_code,
__node_ptr __n, size_type __n_elt = 1);
iterator
_M_insert_multi_node(__node_ptr __hint,
__hash_code __code, __node_ptr __n);
template<typename... _Args>
std::pair<iterator, bool>
_M_emplace(true_type __uks, _Args&&... __args);
template<typename... _Args>
iterator
_M_emplace(false_type __uks, _Args&&... __args)
{ return _M_emplace(cend(), __uks, std::forward<_Args>(__args)...); }
template<typename... _Args>
iterator
_M_emplace(const_iterator, true_type __uks, _Args&&... __args)
{ return _M_emplace(__uks, std::forward<_Args>(__args)...).first; }
template<typename... _Args>
iterator
_M_emplace(const_iterator, false_type __uks, _Args&&... __args);
template<typename _Kt, typename _Arg, typename _NodeGenerator>
std::pair<iterator, bool>
_M_insert_unique(_Kt&&, _Arg&&, const _NodeGenerator&);
template<typename _Kt>
static __conditional_t<
__and_<__is_nothrow_invocable<_Hash&, const key_type&>,
__not_<__is_nothrow_invocable<_Hash&, _Kt>>>::value,
key_type, _Kt&&>
_S_forward_key(_Kt&& __k)
{ return std::forward<_Kt>(__k); }
static const key_type&
_S_forward_key(const key_type& __k)
{ return __k; }
static key_type&&
_S_forward_key(key_type&& __k)
{ return std::move(__k); }
template<typename _Arg, typename _NodeGenerator>
std::pair<iterator, bool>
_M_insert_unique_aux(_Arg&& __arg, const _NodeGenerator& __node_gen)
{
return _M_insert_unique(
_S_forward_key(_ExtractKey{}(std::forward<_Arg>(__arg))),
std::forward<_Arg>(__arg), __node_gen);
}
template<typename _Arg, typename _NodeGenerator>
std::pair<iterator, bool>
_M_insert(_Arg&& __arg, const _NodeGenerator& __node_gen,
true_type )
{
using __to_value
= __detail::_ConvertToValueType<_ExtractKey, value_type>;
return _M_insert_unique_aux(
__to_value{}(std::forward<_Arg>(__arg)), __node_gen);
}
template<typename _Arg, typename _NodeGenerator>
iterator
_M_insert(_Arg&& __arg, const _NodeGenerator& __node_gen,
false_type __uks)
{
using __to_value
= __detail::_ConvertToValueType<_ExtractKey, value_type>;
return _M_insert(cend(),
__to_value{}(std::forward<_Arg>(__arg)), __node_gen, __uks);
}
template<typename _Arg, typename _NodeGenerator>
iterator
_M_insert(const_iterator, _Arg&& __arg,
const _NodeGenerator& __node_gen, true_type __uks)
{
return
_M_insert(std::forward<_Arg>(__arg), __node_gen, __uks).first;
}
template<typename _Arg, typename _NodeGenerator>
iterator
_M_insert(const_iterator, _Arg&&,
const _NodeGenerator&, false_type __uks);
size_type
_M_erase(true_type __uks, const key_type&);
size_type
_M_erase(false_type __uks, const key_type&);
iterator
_M_erase(size_type __bkt, __node_base_ptr __prev_n, __node_ptr __n);
public:
template<typename... _Args>
__ireturn_type
emplace(_Args&&... __args)
{ return _M_emplace(__unique_keys{}, std::forward<_Args>(__args)...); }
template<typename... _Args>
iterator
emplace_hint(const_iterator __hint, _Args&&... __args)
{
return _M_emplace(__hint, __unique_keys{},
std::forward<_Args>(__args)...);
}
iterator
erase(const_iterator);
iterator
erase(iterator __it)
{ return erase(const_iterator(__it)); }
size_type
erase(const key_type& __k)
{ return _M_erase(__unique_keys{}, __k); }
iterator
erase(const_iterator, const_iterator);
void
clear() noexcept;
void rehash(size_type __bkt_count);
insert_return_type
_M_reinsert_node(node_type&& __nh)
{
insert_return_type __ret;
if (__nh.empty())
__ret.position = end();
else
{
do { if (std::__is_constant_evaluated() && !bool(get_allocator() == __nh.get_allocator())) __builtin_unreachable(); } while (false);
const key_type& __k = __nh._M_key();
__hash_code __code = this->_M_hash_code(__k);
size_type __bkt = _M_bucket_index(__code);
if (__node_ptr __n = _M_find_node(__bkt, __k, __code))
{
__ret.node = std::move(__nh);
__ret.position = iterator(__n);
__ret.inserted = false;
}
else
{
__ret.position
= _M_insert_unique_node(__bkt, __code, __nh._M_ptr);
__nh._M_ptr = nullptr;
__ret.inserted = true;
}
}
return __ret;
}
iterator
_M_reinsert_node_multi(const_iterator __hint, node_type&& __nh)
{
if (__nh.empty())
return end();
do { if (std::__is_constant_evaluated() && !bool(get_allocator() == __nh.get_allocator())) __builtin_unreachable(); } while (false);
const key_type& __k = __nh._M_key();
auto __code = this->_M_hash_code(__k);
auto __ret
= _M_insert_multi_node(__hint._M_cur, __code, __nh._M_ptr);
__nh._M_ptr = nullptr;
return __ret;
}
private:
node_type
_M_extract_node(size_t __bkt, __node_base_ptr __prev_n)
{
__node_ptr __n = static_cast<__node_ptr>(__prev_n->_M_nxt);
if (__prev_n == _M_buckets[__bkt])
_M_remove_bucket_begin(__bkt, __n->_M_next(),
__n->_M_nxt ? _M_bucket_index(*__n->_M_next()) : 0);
else if (__n->_M_nxt)
{
size_type __next_bkt = _M_bucket_index(*__n->_M_next());
if (__next_bkt != __bkt)
_M_buckets[__next_bkt] = __prev_n;
}
__prev_n->_M_nxt = __n->_M_nxt;
__n->_M_nxt = nullptr;
--_M_element_count;
return { __n, this->_M_node_allocator() };
}
public:
node_type
extract(const_iterator __pos)
{
size_t __bkt = _M_bucket_index(*__pos._M_cur);
return _M_extract_node(__bkt,
_M_get_previous_node(__bkt, __pos._M_cur));
}
node_type
extract(const _Key& __k)
{
node_type __nh;
__hash_code __code = this->_M_hash_code(__k);
std::size_t __bkt = _M_bucket_index(__code);
if (__node_base_ptr __prev_node = _M_find_before_node(__bkt, __k, __code))
__nh = _M_extract_node(__bkt, __prev_node);
return __nh;
}
template<typename _Compatible_Hashtable>
void
_M_merge_unique(_Compatible_Hashtable& __src)
{
static_assert(is_same_v<typename _Compatible_Hashtable::node_type,
node_type>, "Node types are compatible");
do { if (std::__is_constant_evaluated() && !bool(get_allocator() == __src.get_allocator())) __builtin_unreachable(); } while (false);
auto __n_elt = __src.size();
for (auto __i = __src.cbegin(), __end = __src.cend(); __i != __end;)
{
auto __pos = __i++;
const key_type& __k = _ExtractKey{}(*__pos);
__hash_code __code
= this->_M_hash_code(__src.hash_function(), *__pos._M_cur);
size_type __bkt = _M_bucket_index(__code);
if (_M_find_node(__bkt, __k, __code) == nullptr)
{
auto __nh = __src.extract(__pos);
_M_insert_unique_node(__bkt, __code, __nh._M_ptr, __n_elt);
__nh._M_ptr = nullptr;
__n_elt = 1;
}
else if (__n_elt != 1)
--__n_elt;
}
}
template<typename _Compatible_Hashtable>
void
_M_merge_multi(_Compatible_Hashtable& __src)
{
static_assert(is_same_v<typename _Compatible_Hashtable::node_type,
node_type>, "Node types are compatible");
do { if (std::__is_constant_evaluated() && !bool(get_allocator() == __src.get_allocator())) __builtin_unreachable(); } while (false);
__node_ptr __hint = nullptr;
this->reserve(size() + __src.size());
for (auto __i = __src.cbegin(), __end = __src.cend(); __i != __end;)
{
auto __pos = __i++;
__hash_code __code
= this->_M_hash_code(__src.hash_function(), *__pos._M_cur);
auto __nh = __src.extract(__pos);
__hint = _M_insert_multi_node(__hint, __code, __nh._M_ptr)._M_cur;
__nh._M_ptr = nullptr;
}
}
private:
void _M_rehash_aux(size_type __bkt_count, true_type __uks);
void _M_rehash_aux(size_type __bkt_count, false_type __uks);
void _M_rehash(size_type __bkt_count, const __rehash_state& __state);
};
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_bucket_begin(size_type __bkt) const
-> __node_ptr
{
__node_base_ptr __n = _M_buckets[__bkt];
return __n ? static_cast<__node_ptr>(__n->_M_nxt) : nullptr;
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_Hashtable(size_type __bkt_count_hint,
const _Hash& __h, const _Equal& __eq, const allocator_type& __a)
: _Hashtable(__h, __eq, __a)
{
auto __bkt_count = _M_rehash_policy._M_next_bkt(__bkt_count_hint);
if (__bkt_count > _M_bucket_count)
{
_M_buckets = _M_allocate_buckets(__bkt_count);
_M_bucket_count = __bkt_count;
}
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
template<typename _InputIterator>
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_Hashtable(_InputIterator __f, _InputIterator __l,
size_type __bkt_count_hint,
const _Hash& __h, const _Equal& __eq,
const allocator_type& __a, true_type )
: _Hashtable(__bkt_count_hint, __h, __eq, __a)
{ this->insert(__f, __l); }
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
template<typename _InputIterator>
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_Hashtable(_InputIterator __f, _InputIterator __l,
size_type __bkt_count_hint,
const _Hash& __h, const _Equal& __eq,
const allocator_type& __a, false_type __uks)
: _Hashtable(__h, __eq, __a)
{
auto __nb_elems = __detail::__distance_fw(__f, __l);
auto __bkt_count =
_M_rehash_policy._M_next_bkt(
std::max(_M_rehash_policy._M_bkt_for_elements(__nb_elems),
__bkt_count_hint));
if (__bkt_count > _M_bucket_count)
{
_M_buckets = _M_allocate_buckets(__bkt_count);
_M_bucket_count = __bkt_count;
}
__alloc_node_gen_t __node_gen(*this);
for (; __f != __l; ++__f)
_M_insert(*__f, __node_gen, __uks);
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
operator=(const _Hashtable& __ht)
-> _Hashtable&
{
if (&__ht == this)
return *this;
if (__node_alloc_traits::_S_propagate_on_copy_assign())
{
auto& __this_alloc = this->_M_node_allocator();
auto& __that_alloc = __ht._M_node_allocator();
if (!__node_alloc_traits::_S_always_equal()
&& __this_alloc != __that_alloc)
{
this->_M_deallocate_nodes(_M_begin());
_M_before_begin._M_nxt = nullptr;
_M_deallocate_buckets();
_M_buckets = nullptr;
std::__alloc_on_copy(__this_alloc, __that_alloc);
__hashtable_base::operator=(__ht);
_M_bucket_count = __ht._M_bucket_count;
_M_element_count = __ht._M_element_count;
_M_rehash_policy = __ht._M_rehash_policy;
__alloc_node_gen_t __alloc_node_gen(*this);
try
{
_M_assign(__ht, __alloc_node_gen);
}
catch(...)
{
_M_reset();
throw;
}
return *this;
}
std::__alloc_on_copy(__this_alloc, __that_alloc);
}
_M_assign_elements(__ht);
return *this;
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
template<typename _Ht>
void
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_assign_elements(_Ht&& __ht)
{
__buckets_ptr __former_buckets = nullptr;
std::size_t __former_bucket_count = _M_bucket_count;
const __rehash_state& __former_state = _M_rehash_policy._M_state();
if (_M_bucket_count != __ht._M_bucket_count)
{
__former_buckets = _M_buckets;
_M_buckets = _M_allocate_buckets(__ht._M_bucket_count);
_M_bucket_count = __ht._M_bucket_count;
}
else
__builtin_memset(_M_buckets, 0,
_M_bucket_count * sizeof(__node_base_ptr));
try
{
__hashtable_base::operator=(std::forward<_Ht>(__ht));
_M_element_count = __ht._M_element_count;
_M_rehash_policy = __ht._M_rehash_policy;
__reuse_or_alloc_node_gen_t __roan(_M_begin(), *this);
_M_before_begin._M_nxt = nullptr;
_M_assign(std::forward<_Ht>(__ht), __roan);
if (__former_buckets)
_M_deallocate_buckets(__former_buckets, __former_bucket_count);
}
catch(...)
{
if (__former_buckets)
{
_M_deallocate_buckets();
_M_rehash_policy._M_reset(__former_state);
_M_buckets = __former_buckets;
_M_bucket_count = __former_bucket_count;
}
__builtin_memset(_M_buckets, 0,
_M_bucket_count * sizeof(__node_base_ptr));
throw;
}
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
template<typename _Ht, typename _NodeGenerator>
void
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_assign(_Ht&& __ht, const _NodeGenerator& __node_gen)
{
__buckets_ptr __buckets = nullptr;
if (!_M_buckets)
_M_buckets = __buckets = _M_allocate_buckets(_M_bucket_count);
try
{
if (!__ht._M_before_begin._M_nxt)
return;
__node_ptr __ht_n = __ht._M_begin();
__node_ptr __this_n
= __node_gen(__fwd_value_for<_Ht>(__ht_n->_M_v()));
this->_M_copy_code(*__this_n, *__ht_n);
_M_update_bbegin(__this_n);
__node_ptr __prev_n = __this_n;
for (__ht_n = __ht_n->_M_next(); __ht_n; __ht_n = __ht_n->_M_next())
{
__this_n = __node_gen(__fwd_value_for<_Ht>(__ht_n->_M_v()));
__prev_n->_M_nxt = __this_n;
this->_M_copy_code(*__this_n, *__ht_n);
size_type __bkt = _M_bucket_index(*__this_n);
if (!_M_buckets[__bkt])
_M_buckets[__bkt] = __prev_n;
__prev_n = __this_n;
}
}
catch(...)
{
clear();
if (__buckets)
_M_deallocate_buckets();
throw;
}
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
void
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_reset() noexcept
{
_M_rehash_policy._M_reset();
_M_bucket_count = 1;
_M_single_bucket = nullptr;
_M_buckets = &_M_single_bucket;
_M_before_begin._M_nxt = nullptr;
_M_element_count = 0;
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
void
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_move_assign(_Hashtable&& __ht, true_type)
{
if (__builtin_expect(std::__addressof(__ht) == this, false))
return;
this->_M_deallocate_nodes(_M_begin());
_M_deallocate_buckets();
__hashtable_base::operator=(std::move(__ht));
_M_rehash_policy = __ht._M_rehash_policy;
if (!__ht._M_uses_single_bucket())
_M_buckets = __ht._M_buckets;
else
{
_M_buckets = &_M_single_bucket;
_M_single_bucket = __ht._M_single_bucket;
}
_M_bucket_count = __ht._M_bucket_count;
_M_before_begin._M_nxt = __ht._M_before_begin._M_nxt;
_M_element_count = __ht._M_element_count;
std::__alloc_on_move(this->_M_node_allocator(), __ht._M_node_allocator());
_M_update_bbegin();
__ht._M_reset();
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
void
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_move_assign(_Hashtable&& __ht, false_type)
{
if (__ht._M_node_allocator() == this->_M_node_allocator())
_M_move_assign(std::move(__ht), true_type{});
else
{
_M_assign_elements(std::move(__ht));
__ht.clear();
}
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_Hashtable(const _Hashtable& __ht)
: __hashtable_base(__ht),
__map_base(__ht),
__rehash_base(__ht),
__hashtable_alloc(
__node_alloc_traits::_S_select_on_copy(__ht._M_node_allocator())),
__enable_default_ctor(__ht),
_M_buckets(nullptr),
_M_bucket_count(__ht._M_bucket_count),
_M_element_count(__ht._M_element_count),
_M_rehash_policy(__ht._M_rehash_policy)
{
__alloc_node_gen_t __alloc_node_gen(*this);
_M_assign(__ht, __alloc_node_gen);
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_Hashtable(_Hashtable&& __ht, __node_alloc_type&& __a,
true_type )
noexcept(_S_nothrow_move())
: __hashtable_base(__ht),
__map_base(__ht),
__rehash_base(__ht),
__hashtable_alloc(std::move(__a)),
__enable_default_ctor(__ht),
_M_buckets(__ht._M_buckets),
_M_bucket_count(__ht._M_bucket_count),
_M_before_begin(__ht._M_before_begin._M_nxt),
_M_element_count(__ht._M_element_count),
_M_rehash_policy(__ht._M_rehash_policy)
{
if (__ht._M_uses_single_bucket())
{
_M_buckets = &_M_single_bucket;
_M_single_bucket = __ht._M_single_bucket;
}
_M_update_bbegin();
__ht._M_reset();
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_Hashtable(const _Hashtable& __ht, const allocator_type& __a)
: __hashtable_base(__ht),
__map_base(__ht),
__rehash_base(__ht),
__hashtable_alloc(__node_alloc_type(__a)),
__enable_default_ctor(__ht),
_M_buckets(),
_M_bucket_count(__ht._M_bucket_count),
_M_element_count(__ht._M_element_count),
_M_rehash_policy(__ht._M_rehash_policy)
{
__alloc_node_gen_t __alloc_node_gen(*this);
_M_assign(__ht, __alloc_node_gen);
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_Hashtable(_Hashtable&& __ht, __node_alloc_type&& __a,
false_type )
: __hashtable_base(__ht),
__map_base(__ht),
__rehash_base(__ht),
__hashtable_alloc(std::move(__a)),
__enable_default_ctor(__ht),
_M_buckets(nullptr),
_M_bucket_count(__ht._M_bucket_count),
_M_element_count(__ht._M_element_count),
_M_rehash_policy(__ht._M_rehash_policy)
{
if (__ht._M_node_allocator() == this->_M_node_allocator())
{
if (__ht._M_uses_single_bucket())
{
_M_buckets = &_M_single_bucket;
_M_single_bucket = __ht._M_single_bucket;
}
else
_M_buckets = __ht._M_buckets;
_M_update_bbegin(__ht._M_begin());
__ht._M_reset();
}
else
{
__alloc_node_gen_t __alloc_gen(*this);
using _Fwd_Ht = __conditional_t<
__move_if_noexcept_cond<value_type>::value,
const _Hashtable&, _Hashtable&&>;
_M_assign(std::forward<_Fwd_Ht>(__ht), __alloc_gen);
__ht.clear();
}
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
~_Hashtable() noexcept
{
static_assert(noexcept(declval<const __hash_code_base_access&>()
._M_bucket_index(declval<const __node_value_type&>(),
(std::size_t)0)),
"Cache the hash code or qualify your functors involved"
" in hash code and bucket index computation with noexcept");
clear();
_M_deallocate_buckets();
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
void
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
swap(_Hashtable& __x)
noexcept(__and_<__is_nothrow_swappable<_Hash>,
__is_nothrow_swappable<_Equal>>::value)
{
this->_M_swap(__x);
std::__alloc_on_swap(this->_M_node_allocator(), __x._M_node_allocator());
std::swap(_M_rehash_policy, __x._M_rehash_policy);
if (this->_M_uses_single_bucket())
{
if (!__x._M_uses_single_bucket())
{
_M_buckets = __x._M_buckets;
__x._M_buckets = &__x._M_single_bucket;
}
}
else if (__x._M_uses_single_bucket())
{
__x._M_buckets = _M_buckets;
_M_buckets = &_M_single_bucket;
}
else
std::swap(_M_buckets, __x._M_buckets);
std::swap(_M_bucket_count, __x._M_bucket_count);
std::swap(_M_before_begin._M_nxt, __x._M_before_begin._M_nxt);
std::swap(_M_element_count, __x._M_element_count);
std::swap(_M_single_bucket, __x._M_single_bucket);
_M_update_bbegin();
__x._M_update_bbegin();
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
find(const key_type& __k)
-> iterator
{
if (size() <= __small_size_threshold())
{
for (auto __it = begin(); __it != end(); ++__it)
if (this->_M_key_equals(__k, *__it._M_cur))
return __it;
return end();
}
__hash_code __code = this->_M_hash_code(__k);
std::size_t __bkt = _M_bucket_index(__code);
return iterator(_M_find_node(__bkt, __k, __code));
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
find(const key_type& __k) const
-> const_iterator
{
if (size() <= __small_size_threshold())
{
for (auto __it = begin(); __it != end(); ++__it)
if (this->_M_key_equals(__k, *__it._M_cur))
return __it;
return end();
}
__hash_code __code = this->_M_hash_code(__k);
std::size_t __bkt = _M_bucket_index(__code);
return const_iterator(_M_find_node(__bkt, __k, __code));
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
count(const key_type& __k) const
-> size_type
{
auto __it = find(__k);
if (!__it._M_cur)
return 0;
if (__unique_keys::value)
return 1;
size_type __result = 1;
for (auto __ref = __it++;
__it._M_cur && this->_M_node_equals(*__ref._M_cur, *__it._M_cur);
++__it)
++__result;
return __result;
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
equal_range(const key_type& __k)
-> pair<iterator, iterator>
{
auto __ite = find(__k);
if (!__ite._M_cur)
return { __ite, __ite };
auto __beg = __ite++;
if (__unique_keys::value)
return { __beg, __ite };
while (__ite._M_cur && this->_M_node_equals(*__beg._M_cur, *__ite._M_cur))
++__ite;
return { __beg, __ite };
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
equal_range(const key_type& __k) const
-> pair<const_iterator, const_iterator>
{
auto __ite = find(__k);
if (!__ite._M_cur)
return { __ite, __ite };
auto __beg = __ite++;
if (__unique_keys::value)
return { __beg, __ite };
while (__ite._M_cur && this->_M_node_equals(*__beg._M_cur, *__ite._M_cur))
++__ite;
return { __beg, __ite };
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_find_before_node(const key_type& __k)
-> __node_base_ptr
{
__node_base_ptr __prev_p = &_M_before_begin;
if (!__prev_p->_M_nxt)
return nullptr;
for (__node_ptr __p = static_cast<__node_ptr>(__prev_p->_M_nxt);
__p != nullptr;
__p = __p->_M_next())
{
if (this->_M_key_equals(__k, *__p))
return __prev_p;
__prev_p = __p;
}
return nullptr;
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_find_before_node(size_type __bkt, const key_type& __k,
__hash_code __code) const
-> __node_base_ptr
{
__node_base_ptr __prev_p = _M_buckets[__bkt];
if (!__prev_p)
return nullptr;
for (__node_ptr __p = static_cast<__node_ptr>(__prev_p->_M_nxt);;
__p = __p->_M_next())
{
if (this->_M_equals(__k, __code, *__p))
return __prev_p;
if (!__p->_M_nxt || _M_bucket_index(*__p->_M_next()) != __bkt)
break;
__prev_p = __p;
}
return nullptr;
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
template<typename _Kt>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_find_before_node_tr(size_type __bkt, const _Kt& __k,
__hash_code __code) const
-> __node_base_ptr
{
__node_base_ptr __prev_p = _M_buckets[__bkt];
if (!__prev_p)
return nullptr;
for (__node_ptr __p = static_cast<__node_ptr>(__prev_p->_M_nxt);;
__p = __p->_M_next())
{
if (this->_M_equals_tr(__k, __code, *__p))
return __prev_p;
if (!__p->_M_nxt || _M_bucket_index(*__p->_M_next()) != __bkt)
break;
__prev_p = __p;
}
return nullptr;
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
void
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_insert_bucket_begin(size_type __bkt, __node_ptr __node)
{
if (_M_buckets[__bkt])
{
__node->_M_nxt = _M_buckets[__bkt]->_M_nxt;
_M_buckets[__bkt]->_M_nxt = __node;
}
else
{
__node->_M_nxt = _M_before_begin._M_nxt;
_M_before_begin._M_nxt = __node;
if (__node->_M_nxt)
_M_buckets[_M_bucket_index(*__node->_M_next())] = __node;
_M_buckets[__bkt] = &_M_before_begin;
}
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
void
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_remove_bucket_begin(size_type __bkt, __node_ptr __next,
size_type __next_bkt)
{
if (!__next || __next_bkt != __bkt)
{
if (__next)
_M_buckets[__next_bkt] = _M_buckets[__bkt];
if (&_M_before_begin == _M_buckets[__bkt])
_M_before_begin._M_nxt = __next;
_M_buckets[__bkt] = nullptr;
}
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_get_previous_node(size_type __bkt, __node_ptr __n)
-> __node_base_ptr
{
__node_base_ptr __prev_n = _M_buckets[__bkt];
while (__prev_n->_M_nxt != __n)
__prev_n = __prev_n->_M_nxt;
return __prev_n;
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
template<typename... _Args>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_emplace(true_type , _Args&&... __args)
-> pair<iterator, bool>
{
_Scoped_node __node { this, std::forward<_Args>(__args)... };
const key_type& __k = _ExtractKey{}(__node._M_node->_M_v());
if (size() <= __small_size_threshold())
{
for (auto __it = begin(); __it != end(); ++__it)
if (this->_M_key_equals(__k, *__it._M_cur))
return { __it, false };
}
__hash_code __code = this->_M_hash_code(__k);
size_type __bkt = _M_bucket_index(__code);
if (size() > __small_size_threshold())
if (__node_ptr __p = _M_find_node(__bkt, __k, __code))
return { iterator(__p), false };
auto __pos = _M_insert_unique_node(__bkt, __code, __node._M_node);
__node._M_node = nullptr;
return { __pos, true };
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
template<typename... _Args>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_emplace(const_iterator __hint, false_type ,
_Args&&... __args)
-> iterator
{
_Scoped_node __node { this, std::forward<_Args>(__args)... };
const key_type& __k = _ExtractKey{}(__node._M_node->_M_v());
auto __res = this->_M_compute_hash_code(__hint, __k);
auto __pos
= _M_insert_multi_node(__res.first._M_cur, __res.second,
__node._M_node);
__node._M_node = nullptr;
return __pos;
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_compute_hash_code(const_iterator __hint, const key_type& __k) const
-> pair<const_iterator, __hash_code>
{
if (size() <= __small_size_threshold())
{
if (__hint != cend())
{
for (auto __it = __hint; __it != cend(); ++__it)
if (this->_M_key_equals(__k, *__it._M_cur))
return { __it, this->_M_hash_code(*__it._M_cur) };
}
for (auto __it = cbegin(); __it != __hint; ++__it)
if (this->_M_key_equals(__k, *__it._M_cur))
return { __it, this->_M_hash_code(*__it._M_cur) };
}
return { __hint, this->_M_hash_code(__k) };
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_insert_unique_node(size_type __bkt, __hash_code __code,
__node_ptr __node, size_type __n_elt)
-> iterator
{
const __rehash_state& __saved_state = _M_rehash_policy._M_state();
std::pair<bool, std::size_t> __do_rehash
= _M_rehash_policy._M_need_rehash(_M_bucket_count, _M_element_count,
__n_elt);
if (__do_rehash.first)
{
_M_rehash(__do_rehash.second, __saved_state);
__bkt = _M_bucket_index(__code);
}
this->_M_store_code(*__node, __code);
_M_insert_bucket_begin(__bkt, __node);
++_M_element_count;
return iterator(__node);
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_insert_multi_node(__node_ptr __hint,
__hash_code __code, __node_ptr __node)
-> iterator
{
const __rehash_state& __saved_state = _M_rehash_policy._M_state();
std::pair<bool, std::size_t> __do_rehash
= _M_rehash_policy._M_need_rehash(_M_bucket_count, _M_element_count, 1);
if (__do_rehash.first)
_M_rehash(__do_rehash.second, __saved_state);
this->_M_store_code(*__node, __code);
const key_type& __k = _ExtractKey{}(__node->_M_v());
size_type __bkt = _M_bucket_index(__code);
__node_base_ptr __prev
= __builtin_expect(__hint != nullptr, false)
&& this->_M_equals(__k, __code, *__hint)
? __hint
: _M_find_before_node(__bkt, __k, __code);
if (__prev)
{
__node->_M_nxt = __prev->_M_nxt;
__prev->_M_nxt = __node;
if (__builtin_expect(__prev == __hint, false))
if (__node->_M_nxt
&& !this->_M_equals(__k, __code, *__node->_M_next()))
{
size_type __next_bkt = _M_bucket_index(*__node->_M_next());
if (__next_bkt != __bkt)
_M_buckets[__next_bkt] = __node;
}
}
else
_M_insert_bucket_begin(__bkt, __node);
++_M_element_count;
return iterator(__node);
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
template<typename _Kt, typename _Arg, typename _NodeGenerator>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_insert_unique(_Kt&& __k, _Arg&& __v,
const _NodeGenerator& __node_gen)
-> pair<iterator, bool>
{
if (size() <= __small_size_threshold())
for (auto __it = begin(); __it != end(); ++__it)
if (this->_M_key_equals_tr(__k, *__it._M_cur))
return { __it, false };
__hash_code __code = this->_M_hash_code_tr(__k);
size_type __bkt = _M_bucket_index(__code);
if (size() > __small_size_threshold())
if (__node_ptr __node = _M_find_node_tr(__bkt, __k, __code))
return { iterator(__node), false };
_Scoped_node __node {
__node_builder_t::_S_build(std::forward<_Kt>(__k),
std::forward<_Arg>(__v),
__node_gen),
this
};
auto __pos
= _M_insert_unique_node(__bkt, __code, __node._M_node);
__node._M_node = nullptr;
return { __pos, true };
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
template<typename _Arg, typename _NodeGenerator>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_insert(const_iterator __hint, _Arg&& __v,
const _NodeGenerator& __node_gen,
false_type )
-> iterator
{
_Scoped_node __node{ __node_gen(std::forward<_Arg>(__v)), this };
auto __res = this->_M_compute_hash_code(
__hint, _ExtractKey{}(__node._M_node->_M_v()));
auto __pos
= _M_insert_multi_node(__res.first._M_cur, __res.second,
__node._M_node);
__node._M_node = nullptr;
return __pos;
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
erase(const_iterator __it)
-> iterator
{
__node_ptr __n = __it._M_cur;
std::size_t __bkt = _M_bucket_index(*__n);
__node_base_ptr __prev_n = _M_get_previous_node(__bkt, __n);
return _M_erase(__bkt, __prev_n, __n);
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_erase(size_type __bkt, __node_base_ptr __prev_n, __node_ptr __n)
-> iterator
{
if (__prev_n == _M_buckets[__bkt])
_M_remove_bucket_begin(__bkt, __n->_M_next(),
__n->_M_nxt ? _M_bucket_index(*__n->_M_next()) : 0);
else if (__n->_M_nxt)
{
size_type __next_bkt = _M_bucket_index(*__n->_M_next());
if (__next_bkt != __bkt)
_M_buckets[__next_bkt] = __prev_n;
}
__prev_n->_M_nxt = __n->_M_nxt;
iterator __result(__n->_M_next());
this->_M_deallocate_node(__n);
--_M_element_count;
return __result;
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_erase(true_type , const key_type& __k)
-> size_type
{
__node_base_ptr __prev_n;
__node_ptr __n;
std::size_t __bkt;
if (size() <= __small_size_threshold())
{
__prev_n = _M_find_before_node(__k);
if (!__prev_n)
return 0;
__n = static_cast<__node_ptr>(__prev_n->_M_nxt);
__bkt = _M_bucket_index(*__n);
}
else
{
__hash_code __code = this->_M_hash_code(__k);
__bkt = _M_bucket_index(__code);
__prev_n = _M_find_before_node(__bkt, __k, __code);
if (!__prev_n)
return 0;
__n = static_cast<__node_ptr>(__prev_n->_M_nxt);
}
_M_erase(__bkt, __prev_n, __n);
return 1;
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_erase(false_type , const key_type& __k)
-> size_type
{
std::size_t __bkt;
__node_base_ptr __prev_n;
__node_ptr __n;
if (size() <= __small_size_threshold())
{
__prev_n = _M_find_before_node(__k);
if (!__prev_n)
return 0;
__n = static_cast<__node_ptr>(__prev_n->_M_nxt);
__bkt = _M_bucket_index(*__n);
}
else
{
__hash_code __code = this->_M_hash_code(__k);
__bkt = _M_bucket_index(__code);
__prev_n = _M_find_before_node(__bkt, __k, __code);
if (!__prev_n)
return 0;
__n = static_cast<__node_ptr>(__prev_n->_M_nxt);
}
__node_ptr __n_last = __n->_M_next();
while (__n_last && this->_M_node_equals(*__n, *__n_last))
__n_last = __n_last->_M_next();
std::size_t __n_last_bkt = __n_last ? _M_bucket_index(*__n_last) : __bkt;
size_type __result = 0;
do
{
__node_ptr __p = __n->_M_next();
this->_M_deallocate_node(__n);
__n = __p;
++__result;
}
while (__n != __n_last);
_M_element_count -= __result;
if (__prev_n == _M_buckets[__bkt])
_M_remove_bucket_begin(__bkt, __n_last, __n_last_bkt);
else if (__n_last_bkt != __bkt)
_M_buckets[__n_last_bkt] = __prev_n;
__prev_n->_M_nxt = __n_last;
return __result;
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
erase(const_iterator __first, const_iterator __last)
-> iterator
{
__node_ptr __n = __first._M_cur;
__node_ptr __last_n = __last._M_cur;
if (__n == __last_n)
return iterator(__n);
std::size_t __bkt = _M_bucket_index(*__n);
__node_base_ptr __prev_n = _M_get_previous_node(__bkt, __n);
bool __is_bucket_begin = __n == _M_bucket_begin(__bkt);
std::size_t __n_bkt = __bkt;
for (;;)
{
do
{
__node_ptr __tmp = __n;
__n = __n->_M_next();
this->_M_deallocate_node(__tmp);
--_M_element_count;
if (!__n)
break;
__n_bkt = _M_bucket_index(*__n);
}
while (__n != __last_n && __n_bkt == __bkt);
if (__is_bucket_begin)
_M_remove_bucket_begin(__bkt, __n, __n_bkt);
if (__n == __last_n)
break;
__is_bucket_begin = true;
__bkt = __n_bkt;
}
if (__n && (__n_bkt != __bkt || __is_bucket_begin))
_M_buckets[__n_bkt] = __prev_n;
__prev_n->_M_nxt = __n;
return iterator(__n);
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
void
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
clear() noexcept
{
this->_M_deallocate_nodes(_M_begin());
__builtin_memset(_M_buckets, 0,
_M_bucket_count * sizeof(__node_base_ptr));
_M_element_count = 0;
_M_before_begin._M_nxt = nullptr;
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
void
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
rehash(size_type __bkt_count)
{
const __rehash_state& __saved_state = _M_rehash_policy._M_state();
__bkt_count
= std::max(_M_rehash_policy._M_bkt_for_elements(_M_element_count + 1),
__bkt_count);
__bkt_count = _M_rehash_policy._M_next_bkt(__bkt_count);
if (__bkt_count != _M_bucket_count)
_M_rehash(__bkt_count, __saved_state);
else
_M_rehash_policy._M_reset(__saved_state);
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
void
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_rehash(size_type __bkt_count, const __rehash_state& __state)
{
try
{
_M_rehash_aux(__bkt_count, __unique_keys{});
}
catch(...)
{
_M_rehash_policy._M_reset(__state);
throw;
}
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
void
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_rehash_aux(size_type __bkt_count, true_type )
{
__buckets_ptr __new_buckets = _M_allocate_buckets(__bkt_count);
__node_ptr __p = _M_begin();
_M_before_begin._M_nxt = nullptr;
std::size_t __bbegin_bkt = 0;
while (__p)
{
__node_ptr __next = __p->_M_next();
std::size_t __bkt
= __hash_code_base::_M_bucket_index(*__p, __bkt_count);
if (!__new_buckets[__bkt])
{
__p->_M_nxt = _M_before_begin._M_nxt;
_M_before_begin._M_nxt = __p;
__new_buckets[__bkt] = &_M_before_begin;
if (__p->_M_nxt)
__new_buckets[__bbegin_bkt] = __p;
__bbegin_bkt = __bkt;
}
else
{
__p->_M_nxt = __new_buckets[__bkt]->_M_nxt;
__new_buckets[__bkt]->_M_nxt = __p;
}
__p = __next;
}
_M_deallocate_buckets();
_M_bucket_count = __bkt_count;
_M_buckets = __new_buckets;
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
void
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_rehash_aux(size_type __bkt_count, false_type )
{
__buckets_ptr __new_buckets = _M_allocate_buckets(__bkt_count);
__node_ptr __p = _M_begin();
_M_before_begin._M_nxt = nullptr;
std::size_t __bbegin_bkt = 0;
std::size_t __prev_bkt = 0;
__node_ptr __prev_p = nullptr;
bool __check_bucket = false;
while (__p)
{
__node_ptr __next = __p->_M_next();
std::size_t __bkt
= __hash_code_base::_M_bucket_index(*__p, __bkt_count);
if (__prev_p && __prev_bkt == __bkt)
{
__p->_M_nxt = __prev_p->_M_nxt;
__prev_p->_M_nxt = __p;
__check_bucket = true;
}
else
{
if (__check_bucket)
{
if (__prev_p->_M_nxt)
{
std::size_t __next_bkt
= __hash_code_base::_M_bucket_index(
*__prev_p->_M_next(), __bkt_count);
if (__next_bkt != __prev_bkt)
__new_buckets[__next_bkt] = __prev_p;
}
__check_bucket = false;
}
if (!__new_buckets[__bkt])
{
__p->_M_nxt = _M_before_begin._M_nxt;
_M_before_begin._M_nxt = __p;
__new_buckets[__bkt] = &_M_before_begin;
if (__p->_M_nxt)
__new_buckets[__bbegin_bkt] = __p;
__bbegin_bkt = __bkt;
}
else
{
__p->_M_nxt = __new_buckets[__bkt]->_M_nxt;
__new_buckets[__bkt]->_M_nxt = __p;
}
}
__prev_p = __p;
__prev_bkt = __bkt;
__p = __next;
}
if (__check_bucket && __prev_p->_M_nxt)
{
std::size_t __next_bkt
= __hash_code_base::_M_bucket_index(*__prev_p->_M_next(),
__bkt_count);
if (__next_bkt != __prev_bkt)
__new_buckets[__next_bkt] = __prev_p;
}
_M_deallocate_buckets();
_M_bucket_count = __bkt_count;
_M_buckets = __new_buckets;
}
template<typename, typename, typename> class _Hash_merge_helper { };
template<typename _Hash>
using _RequireNotAllocatorOrIntegral
= __enable_if_t<!__or_<is_integral<_Hash>, __is_allocator<_Hash>>::value>;
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<bool _Cache>
using __umap_traits = __detail::_Hashtable_traits<_Cache, false, true>;
template<typename _Key,
typename _Tp,
typename _Hash = hash<_Key>,
typename _Pred = std::equal_to<_Key>,
typename _Alloc = std::allocator<std::pair<const _Key, _Tp> >,
typename _Tr = __umap_traits<__cache_default<_Key, _Hash>::value>>
using __umap_hashtable = _Hashtable<_Key, std::pair<const _Key, _Tp>,
_Alloc, __detail::_Select1st,
_Pred, _Hash,
__detail::_Mod_range_hashing,
__detail::_Default_ranged_hash,
__detail::_Prime_rehash_policy, _Tr>;
template<bool _Cache>
using __ummap_traits = __detail::_Hashtable_traits<_Cache, false, false>;
template<typename _Key,
typename _Tp,
typename _Hash = hash<_Key>,
typename _Pred = std::equal_to<_Key>,
typename _Alloc = std::allocator<std::pair<const _Key, _Tp> >,
typename _Tr = __ummap_traits<__cache_default<_Key, _Hash>::value>>
using __ummap_hashtable = _Hashtable<_Key, std::pair<const _Key, _Tp>,
_Alloc, __detail::_Select1st,
_Pred, _Hash,
__detail::_Mod_range_hashing,
__detail::_Default_ranged_hash,
__detail::_Prime_rehash_policy, _Tr>;
template<class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
class unordered_multimap;
template<typename _Key, typename _Tp,
typename _Hash = hash<_Key>,
typename _Pred = equal_to<_Key>,
typename _Alloc = allocator<std::pair<const _Key, _Tp>>>
class unordered_map
{
typedef __umap_hashtable<_Key, _Tp, _Hash, _Pred, _Alloc> _Hashtable;
_Hashtable _M_h;
public:
typedef typename _Hashtable::key_type key_type;
typedef typename _Hashtable::value_type value_type;
typedef typename _Hashtable::mapped_type mapped_type;
typedef typename _Hashtable::hasher hasher;
typedef typename _Hashtable::key_equal key_equal;
typedef typename _Hashtable::allocator_type allocator_type;
typedef typename _Hashtable::pointer pointer;
typedef typename _Hashtable::const_pointer const_pointer;
typedef typename _Hashtable::reference reference;
typedef typename _Hashtable::const_reference const_reference;
typedef typename _Hashtable::iterator iterator;
typedef typename _Hashtable::const_iterator const_iterator;
typedef typename _Hashtable::local_iterator local_iterator;
typedef typename _Hashtable::const_local_iterator const_local_iterator;
typedef typename _Hashtable::size_type size_type;
typedef typename _Hashtable::difference_type difference_type;
using node_type = typename _Hashtable::node_type;
using insert_return_type = typename _Hashtable::insert_return_type;
unordered_map() = default;
explicit
unordered_map(size_type __n,
const hasher& __hf = hasher(),
const key_equal& __eql = key_equal(),
const allocator_type& __a = allocator_type())
: _M_h(__n, __hf, __eql, __a)
{ }
template<typename _InputIterator>
unordered_map(_InputIterator __first, _InputIterator __last,
size_type __n = 0,
const hasher& __hf = hasher(),
const key_equal& __eql = key_equal(),
const allocator_type& __a = allocator_type())
: _M_h(__first, __last, __n, __hf, __eql, __a)
{ }
unordered_map(const unordered_map&) = default;
unordered_map(unordered_map&&) = default;
explicit
unordered_map(const allocator_type& __a)
: _M_h(__a)
{ }
unordered_map(const unordered_map& __umap,
const allocator_type& __a)
: _M_h(__umap._M_h, __a)
{ }
unordered_map(unordered_map&& __umap,
const allocator_type& __a)
noexcept( noexcept(_Hashtable(std::move(__umap._M_h), __a)) )
: _M_h(std::move(__umap._M_h), __a)
{ }
unordered_map(initializer_list<value_type> __l,
size_type __n = 0,
const hasher& __hf = hasher(),
const key_equal& __eql = key_equal(),
const allocator_type& __a = allocator_type())
: _M_h(__l, __n, __hf, __eql, __a)
{ }
unordered_map(size_type __n, const allocator_type& __a)
: unordered_map(__n, hasher(), key_equal(), __a)
{ }
unordered_map(size_type __n, const hasher& __hf,
const allocator_type& __a)
: unordered_map(__n, __hf, key_equal(), __a)
{ }
template<typename _InputIterator>
unordered_map(_InputIterator __first, _InputIterator __last,
size_type __n,
const allocator_type& __a)
: unordered_map(__first, __last, __n, hasher(), key_equal(), __a)
{ }
template<typename _InputIterator>
unordered_map(_InputIterator __first, _InputIterator __last,
size_type __n, const hasher& __hf,
const allocator_type& __a)
: unordered_map(__first, __last, __n, __hf, key_equal(), __a)
{ }
unordered_map(initializer_list<value_type> __l,
size_type __n,
const allocator_type& __a)
: unordered_map(__l, __n, hasher(), key_equal(), __a)
{ }
unordered_map(initializer_list<value_type> __l,
size_type __n, const hasher& __hf,
const allocator_type& __a)
: unordered_map(__l, __n, __hf, key_equal(), __a)
{ }
unordered_map&
operator=(const unordered_map&) = default;
unordered_map&
operator=(unordered_map&&) = default;
unordered_map&
operator=(initializer_list<value_type> __l)
{
_M_h = __l;
return *this;
}
allocator_type
get_allocator() const noexcept
{ return _M_h.get_allocator(); }
[[__nodiscard__]] bool
empty() const noexcept
{ return _M_h.empty(); }
size_type
size() const noexcept
{ return _M_h.size(); }
size_type
max_size() const noexcept
{ return _M_h.max_size(); }
iterator
begin() noexcept
{ return _M_h.begin(); }
const_iterator
begin() const noexcept
{ return _M_h.begin(); }
const_iterator
cbegin() const noexcept
{ return _M_h.begin(); }
iterator
end() noexcept
{ return _M_h.end(); }
const_iterator
end() const noexcept
{ return _M_h.end(); }
const_iterator
cend() const noexcept
{ return _M_h.end(); }
template<typename... _Args>
std::pair<iterator, bool>
emplace(_Args&&... __args)
{ return _M_h.emplace(std::forward<_Args>(__args)...); }
template<typename... _Args>
iterator
emplace_hint(const_iterator __pos, _Args&&... __args)
{ return _M_h.emplace_hint(__pos, std::forward<_Args>(__args)...); }
node_type
extract(const_iterator __pos)
{
do { if (std::__is_constant_evaluated() && !bool(__pos != end())) __builtin_unreachable(); } while (false);
return _M_h.extract(__pos);
}
node_type
extract(const key_type& __key)
{ return _M_h.extract(__key); }
insert_return_type
insert(node_type&& __nh)
{ return _M_h._M_reinsert_node(std::move(__nh)); }
iterator
insert(const_iterator, node_type&& __nh)
{ return _M_h._M_reinsert_node(std::move(__nh)).position; }
template <typename... _Args>
pair<iterator, bool>
try_emplace(const key_type& __k, _Args&&... __args)
{
return _M_h.try_emplace(cend(), __k, std::forward<_Args>(__args)...);
}
template <typename... _Args>
pair<iterator, bool>
try_emplace(key_type&& __k, _Args&&... __args)
{
return _M_h.try_emplace(cend(), std::move(__k),
std::forward<_Args>(__args)...);
}
template <typename... _Args>
iterator
try_emplace(const_iterator __hint, const key_type& __k,
_Args&&... __args)
{
return _M_h.try_emplace(__hint, __k,
std::forward<_Args>(__args)...).first;
}
template <typename... _Args>
iterator
try_emplace(const_iterator __hint, key_type&& __k, _Args&&... __args)
{
return _M_h.try_emplace(__hint, std::move(__k),
std::forward<_Args>(__args)...).first;
}
std::pair<iterator, bool>
insert(const value_type& __x)
{ return _M_h.insert(__x); }
std::pair<iterator, bool>
insert(value_type&& __x)
{ return _M_h.insert(std::move(__x)); }
template<typename _Pair>
__enable_if_t<is_constructible<value_type, _Pair&&>::value,
pair<iterator, bool>>
insert(_Pair&& __x)
{ return _M_h.emplace(std::forward<_Pair>(__x)); }
iterator
insert(const_iterator __hint, const value_type& __x)
{ return _M_h.insert(__hint, __x); }
iterator
insert(const_iterator __hint, value_type&& __x)
{ return _M_h.insert(__hint, std::move(__x)); }
template<typename _Pair>
__enable_if_t<is_constructible<value_type, _Pair&&>::value, iterator>
insert(const_iterator __hint, _Pair&& __x)
{ return _M_h.emplace_hint(__hint, std::forward<_Pair>(__x)); }
template<typename _InputIterator>
void
insert(_InputIterator __first, _InputIterator __last)
{ _M_h.insert(__first, __last); }
void
insert(initializer_list<value_type> __l)
{ _M_h.insert(__l); }
template <typename _Obj>
pair<iterator, bool>
insert_or_assign(const key_type& __k, _Obj&& __obj)
{
auto __ret = _M_h.try_emplace(cend(), __k,
std::forward<_Obj>(__obj));
if (!__ret.second)
__ret.first->second = std::forward<_Obj>(__obj);
return __ret;
}
template <typename _Obj>
pair<iterator, bool>
insert_or_assign(key_type&& __k, _Obj&& __obj)
{
auto __ret = _M_h.try_emplace(cend(), std::move(__k),
std::forward<_Obj>(__obj));
if (!__ret.second)
__ret.first->second = std::forward<_Obj>(__obj);
return __ret;
}
template <typename _Obj>
iterator
insert_or_assign(const_iterator __hint, const key_type& __k,
_Obj&& __obj)
{
auto __ret = _M_h.try_emplace(__hint, __k, std::forward<_Obj>(__obj));
if (!__ret.second)
__ret.first->second = std::forward<_Obj>(__obj);
return __ret.first;
}
template <typename _Obj>
iterator
insert_or_assign(const_iterator __hint, key_type&& __k, _Obj&& __obj)
{
auto __ret = _M_h.try_emplace(__hint, std::move(__k),
std::forward<_Obj>(__obj));
if (!__ret.second)
__ret.first->second = std::forward<_Obj>(__obj);
return __ret.first;
}
iterator
erase(const_iterator __position)
{ return _M_h.erase(__position); }
iterator
erase(iterator __position)
{ return _M_h.erase(__position); }
size_type
erase(const key_type& __x)
{ return _M_h.erase(__x); }
iterator
erase(const_iterator __first, const_iterator __last)
{ return _M_h.erase(__first, __last); }
void
clear() noexcept
{ _M_h.clear(); }
void
swap(unordered_map& __x)
noexcept( noexcept(_M_h.swap(__x._M_h)) )
{ _M_h.swap(__x._M_h); }
template<typename, typename, typename>
friend class std::_Hash_merge_helper;
template<typename _H2, typename _P2>
void
merge(unordered_map<_Key, _Tp, _H2, _P2, _Alloc>& __source)
{
using _Merge_helper = _Hash_merge_helper<unordered_map, _H2, _P2>;
_M_h._M_merge_unique(_Merge_helper::_S_get_table(__source));
}
template<typename _H2, typename _P2>
void
merge(unordered_map<_Key, _Tp, _H2, _P2, _Alloc>&& __source)
{ merge(__source); }
template<typename _H2, typename _P2>
void
merge(unordered_multimap<_Key, _Tp, _H2, _P2, _Alloc>& __source)
{
using _Merge_helper = _Hash_merge_helper<unordered_map, _H2, _P2>;
_M_h._M_merge_unique(_Merge_helper::_S_get_table(__source));
}
template<typename _H2, typename _P2>
void
merge(unordered_multimap<_Key, _Tp, _H2, _P2, _Alloc>&& __source)
{ merge(__source); }
hasher
hash_function() const
{ return _M_h.hash_function(); }
key_equal
key_eq() const
{ return _M_h.key_eq(); }
iterator
find(const key_type& __x)
{ return _M_h.find(__x); }
const_iterator
find(const key_type& __x) const
{ return _M_h.find(__x); }
size_type
count(const key_type& __x) const
{ return _M_h.count(__x); }
std::pair<iterator, iterator>
equal_range(const key_type& __x)
{ return _M_h.equal_range(__x); }
std::pair<const_iterator, const_iterator>
equal_range(const key_type& __x) const
{ return _M_h.equal_range(__x); }
mapped_type&
operator[](const key_type& __k)
{ return _M_h[__k]; }
mapped_type&
operator[](key_type&& __k)
{ return _M_h[std::move(__k)]; }
mapped_type&
at(const key_type& __k)
{ return _M_h.at(__k); }
const mapped_type&
at(const key_type& __k) const
{ return _M_h.at(__k); }
size_type
bucket_count() const noexcept
{ return _M_h.bucket_count(); }
size_type
max_bucket_count() const noexcept
{ return _M_h.max_bucket_count(); }
size_type
bucket_size(size_type __n) const
{ return _M_h.bucket_size(__n); }
size_type
bucket(const key_type& __key) const
{ return _M_h.bucket(__key); }
local_iterator
begin(size_type __n)
{ return _M_h.begin(__n); }
const_local_iterator
begin(size_type __n) const
{ return _M_h.begin(__n); }
const_local_iterator
cbegin(size_type __n) const
{ return _M_h.cbegin(__n); }
local_iterator
end(size_type __n)
{ return _M_h.end(__n); }
const_local_iterator
end(size_type __n) const
{ return _M_h.end(__n); }
const_local_iterator
cend(size_type __n) const
{ return _M_h.cend(__n); }
float
load_factor() const noexcept
{ return _M_h.load_factor(); }
float
max_load_factor() const noexcept
{ return _M_h.max_load_factor(); }
void
max_load_factor(float __z)
{ _M_h.max_load_factor(__z); }
void
rehash(size_type __n)
{ _M_h.rehash(__n); }
void
reserve(size_type __n)
{ _M_h.reserve(__n); }
template<typename _Key1, typename _Tp1, typename _Hash1, typename _Pred1,
typename _Alloc1>
friend bool
operator==(const unordered_map<_Key1, _Tp1, _Hash1, _Pred1, _Alloc1>&,
const unordered_map<_Key1, _Tp1, _Hash1, _Pred1, _Alloc1>&);
};
template<typename _InputIterator,
typename _Hash = hash<__iter_key_t<_InputIterator>>,
typename _Pred = equal_to<__iter_key_t<_InputIterator>>,
typename _Allocator = allocator<__iter_to_alloc_t<_InputIterator>>,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireNotAllocatorOrIntegral<_Hash>,
typename = _RequireNotAllocator<_Pred>,
typename = _RequireAllocator<_Allocator>>
unordered_map(_InputIterator, _InputIterator,
typename unordered_map<int, int>::size_type = {},
_Hash = _Hash(), _Pred = _Pred(), _Allocator = _Allocator())
-> unordered_map<__iter_key_t<_InputIterator>,
__iter_val_t<_InputIterator>,
_Hash, _Pred, _Allocator>;
template<typename _Key, typename _Tp, typename _Hash = hash<_Key>,
typename _Pred = equal_to<_Key>,
typename _Allocator = allocator<pair<const _Key, _Tp>>,
typename = _RequireNotAllocatorOrIntegral<_Hash>,
typename = _RequireNotAllocator<_Pred>,
typename = _RequireAllocator<_Allocator>>
unordered_map(initializer_list<pair<_Key, _Tp>>,
typename unordered_map<int, int>::size_type = {},
_Hash = _Hash(), _Pred = _Pred(), _Allocator = _Allocator())
-> unordered_map<_Key, _Tp, _Hash, _Pred, _Allocator>;
template<typename _InputIterator, typename _Allocator,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireAllocator<_Allocator>>
unordered_map(_InputIterator, _InputIterator,
typename unordered_map<int, int>::size_type, _Allocator)
-> unordered_map<__iter_key_t<_InputIterator>,
__iter_val_t<_InputIterator>,
hash<__iter_key_t<_InputIterator>>,
equal_to<__iter_key_t<_InputIterator>>,
_Allocator>;
template<typename _InputIterator, typename _Allocator,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireAllocator<_Allocator>>
unordered_map(_InputIterator, _InputIterator, _Allocator)
-> unordered_map<__iter_key_t<_InputIterator>,
__iter_val_t<_InputIterator>,
hash<__iter_key_t<_InputIterator>>,
equal_to<__iter_key_t<_InputIterator>>,
_Allocator>;
template<typename _InputIterator, typename _Hash, typename _Allocator,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireNotAllocatorOrIntegral<_Hash>,
typename = _RequireAllocator<_Allocator>>
unordered_map(_InputIterator, _InputIterator,
typename unordered_map<int, int>::size_type,
_Hash, _Allocator)
-> unordered_map<__iter_key_t<_InputIterator>,
__iter_val_t<_InputIterator>, _Hash,
equal_to<__iter_key_t<_InputIterator>>, _Allocator>;
template<typename _Key, typename _Tp, typename _Allocator,
typename = _RequireAllocator<_Allocator>>
unordered_map(initializer_list<pair<_Key, _Tp>>,
typename unordered_map<int, int>::size_type,
_Allocator)
-> unordered_map<_Key, _Tp, hash<_Key>, equal_to<_Key>, _Allocator>;
template<typename _Key, typename _Tp, typename _Allocator,
typename = _RequireAllocator<_Allocator>>
unordered_map(initializer_list<pair<_Key, _Tp>>, _Allocator)
-> unordered_map<_Key, _Tp, hash<_Key>, equal_to<_Key>, _Allocator>;
template<typename _Key, typename _Tp, typename _Hash, typename _Allocator,
typename = _RequireNotAllocatorOrIntegral<_Hash>,
typename = _RequireAllocator<_Allocator>>
unordered_map(initializer_list<pair<_Key, _Tp>>,
typename unordered_map<int, int>::size_type,
_Hash, _Allocator)
-> unordered_map<_Key, _Tp, _Hash, equal_to<_Key>, _Allocator>;
template<typename _Key, typename _Tp,
typename _Hash = hash<_Key>,
typename _Pred = equal_to<_Key>,
typename _Alloc = allocator<std::pair<const _Key, _Tp>>>
class unordered_multimap
{
typedef __ummap_hashtable<_Key, _Tp, _Hash, _Pred, _Alloc> _Hashtable;
_Hashtable _M_h;
public:
typedef typename _Hashtable::key_type key_type;
typedef typename _Hashtable::value_type value_type;
typedef typename _Hashtable::mapped_type mapped_type;
typedef typename _Hashtable::hasher hasher;
typedef typename _Hashtable::key_equal key_equal;
typedef typename _Hashtable::allocator_type allocator_type;
typedef typename _Hashtable::pointer pointer;
typedef typename _Hashtable::const_pointer const_pointer;
typedef typename _Hashtable::reference reference;
typedef typename _Hashtable::const_reference const_reference;
typedef typename _Hashtable::iterator iterator;
typedef typename _Hashtable::const_iterator const_iterator;
typedef typename _Hashtable::local_iterator local_iterator;
typedef typename _Hashtable::const_local_iterator const_local_iterator;
typedef typename _Hashtable::size_type size_type;
typedef typename _Hashtable::difference_type difference_type;
using node_type = typename _Hashtable::node_type;
unordered_multimap() = default;
explicit
unordered_multimap(size_type __n,
const hasher& __hf = hasher(),
const key_equal& __eql = key_equal(),
const allocator_type& __a = allocator_type())
: _M_h(__n, __hf, __eql, __a)
{ }
template<typename _InputIterator>
unordered_multimap(_InputIterator __first, _InputIterator __last,
size_type __n = 0,
const hasher& __hf = hasher(),
const key_equal& __eql = key_equal(),
const allocator_type& __a = allocator_type())
: _M_h(__first, __last, __n, __hf, __eql, __a)
{ }
unordered_multimap(const unordered_multimap&) = default;
unordered_multimap(unordered_multimap&&) = default;
explicit
unordered_multimap(const allocator_type& __a)
: _M_h(__a)
{ }
unordered_multimap(const unordered_multimap& __ummap,
const allocator_type& __a)
: _M_h(__ummap._M_h, __a)
{ }
unordered_multimap(unordered_multimap&& __ummap,
const allocator_type& __a)
noexcept( noexcept(_Hashtable(std::move(__ummap._M_h), __a)) )
: _M_h(std::move(__ummap._M_h), __a)
{ }
unordered_multimap(initializer_list<value_type> __l,
size_type __n = 0,
const hasher& __hf = hasher(),
const key_equal& __eql = key_equal(),
const allocator_type& __a = allocator_type())
: _M_h(__l, __n, __hf, __eql, __a)
{ }
unordered_multimap(size_type __n, const allocator_type& __a)
: unordered_multimap(__n, hasher(), key_equal(), __a)
{ }
unordered_multimap(size_type __n, const hasher& __hf,
const allocator_type& __a)
: unordered_multimap(__n, __hf, key_equal(), __a)
{ }
template<typename _InputIterator>
unordered_multimap(_InputIterator __first, _InputIterator __last,
size_type __n,
const allocator_type& __a)
: unordered_multimap(__first, __last, __n, hasher(), key_equal(), __a)
{ }
template<typename _InputIterator>
unordered_multimap(_InputIterator __first, _InputIterator __last,
size_type __n, const hasher& __hf,
const allocator_type& __a)
: unordered_multimap(__first, __last, __n, __hf, key_equal(), __a)
{ }
unordered_multimap(initializer_list<value_type> __l,
size_type __n,
const allocator_type& __a)
: unordered_multimap(__l, __n, hasher(), key_equal(), __a)
{ }
unordered_multimap(initializer_list<value_type> __l,
size_type __n, const hasher& __hf,
const allocator_type& __a)
: unordered_multimap(__l, __n, __hf, key_equal(), __a)
{ }
unordered_multimap&
operator=(const unordered_multimap&) = default;
unordered_multimap&
operator=(unordered_multimap&&) = default;
unordered_multimap&
operator=(initializer_list<value_type> __l)
{
_M_h = __l;
return *this;
}
allocator_type
get_allocator() const noexcept
{ return _M_h.get_allocator(); }
[[__nodiscard__]] bool
empty() const noexcept
{ return _M_h.empty(); }
size_type
size() const noexcept
{ return _M_h.size(); }
size_type
max_size() const noexcept
{ return _M_h.max_size(); }
iterator
begin() noexcept
{ return _M_h.begin(); }
const_iterator
begin() const noexcept
{ return _M_h.begin(); }
const_iterator
cbegin() const noexcept
{ return _M_h.begin(); }
iterator
end() noexcept
{ return _M_h.end(); }
const_iterator
end() const noexcept
{ return _M_h.end(); }
const_iterator
cend() const noexcept
{ return _M_h.end(); }
template<typename... _Args>
iterator
emplace(_Args&&... __args)
{ return _M_h.emplace(std::forward<_Args>(__args)...); }
template<typename... _Args>
iterator
emplace_hint(const_iterator __pos, _Args&&... __args)
{ return _M_h.emplace_hint(__pos, std::forward<_Args>(__args)...); }
iterator
insert(const value_type& __x)
{ return _M_h.insert(__x); }
iterator
insert(value_type&& __x)
{ return _M_h.insert(std::move(__x)); }
template<typename _Pair>
__enable_if_t<is_constructible<value_type, _Pair&&>::value, iterator>
insert(_Pair&& __x)
{ return _M_h.emplace(std::forward<_Pair>(__x)); }
iterator
insert(const_iterator __hint, const value_type& __x)
{ return _M_h.insert(__hint, __x); }
iterator
insert(const_iterator __hint, value_type&& __x)
{ return _M_h.insert(__hint, std::move(__x)); }
template<typename _Pair>
__enable_if_t<is_constructible<value_type, _Pair&&>::value, iterator>
insert(const_iterator __hint, _Pair&& __x)
{ return _M_h.emplace_hint(__hint, std::forward<_Pair>(__x)); }
template<typename _InputIterator>
void
insert(_InputIterator __first, _InputIterator __last)
{ _M_h.insert(__first, __last); }
void
insert(initializer_list<value_type> __l)
{ _M_h.insert(__l); }
node_type
extract(const_iterator __pos)
{
do { if (std::__is_constant_evaluated() && !bool(__pos != end())) __builtin_unreachable(); } while (false);
return _M_h.extract(__pos);
}
node_type
extract(const key_type& __key)
{ return _M_h.extract(__key); }
iterator
insert(node_type&& __nh)
{ return _M_h._M_reinsert_node_multi(cend(), std::move(__nh)); }
iterator
insert(const_iterator __hint, node_type&& __nh)
{ return _M_h._M_reinsert_node_multi(__hint, std::move(__nh)); }
iterator
erase(const_iterator __position)
{ return _M_h.erase(__position); }
iterator
erase(iterator __position)
{ return _M_h.erase(__position); }
size_type
erase(const key_type& __x)
{ return _M_h.erase(__x); }
iterator
erase(const_iterator __first, const_iterator __last)
{ return _M_h.erase(__first, __last); }
void
clear() noexcept
{ _M_h.clear(); }
void
swap(unordered_multimap& __x)
noexcept( noexcept(_M_h.swap(__x._M_h)) )
{ _M_h.swap(__x._M_h); }
template<typename, typename, typename>
friend class std::_Hash_merge_helper;
template<typename _H2, typename _P2>
void
merge(unordered_multimap<_Key, _Tp, _H2, _P2, _Alloc>& __source)
{
using _Merge_helper
= _Hash_merge_helper<unordered_multimap, _H2, _P2>;
_M_h._M_merge_multi(_Merge_helper::_S_get_table(__source));
}
template<typename _H2, typename _P2>
void
merge(unordered_multimap<_Key, _Tp, _H2, _P2, _Alloc>&& __source)
{ merge(__source); }
template<typename _H2, typename _P2>
void
merge(unordered_map<_Key, _Tp, _H2, _P2, _Alloc>& __source)
{
using _Merge_helper
= _Hash_merge_helper<unordered_multimap, _H2, _P2>;
_M_h._M_merge_multi(_Merge_helper::_S_get_table(__source));
}
template<typename _H2, typename _P2>
void
merge(unordered_map<_Key, _Tp, _H2, _P2, _Alloc>&& __source)
{ merge(__source); }
hasher
hash_function() const
{ return _M_h.hash_function(); }
key_equal
key_eq() const
{ return _M_h.key_eq(); }
iterator
find(const key_type& __x)
{ return _M_h.find(__x); }
const_iterator
find(const key_type& __x) const
{ return _M_h.find(__x); }
size_type
count(const key_type& __x) const
{ return _M_h.count(__x); }
std::pair<iterator, iterator>
equal_range(const key_type& __x)
{ return _M_h.equal_range(__x); }
std::pair<const_iterator, const_iterator>
equal_range(const key_type& __x) const
{ return _M_h.equal_range(__x); }
size_type
bucket_count() const noexcept
{ return _M_h.bucket_count(); }
size_type
max_bucket_count() const noexcept
{ return _M_h.max_bucket_count(); }
size_type
bucket_size(size_type __n) const
{ return _M_h.bucket_size(__n); }
size_type
bucket(const key_type& __key) const
{ return _M_h.bucket(__key); }
local_iterator
begin(size_type __n)
{ return _M_h.begin(__n); }
const_local_iterator
begin(size_type __n) const
{ return _M_h.begin(__n); }
const_local_iterator
cbegin(size_type __n) const
{ return _M_h.cbegin(__n); }
local_iterator
end(size_type __n)
{ return _M_h.end(__n); }
const_local_iterator
end(size_type __n) const
{ return _M_h.end(__n); }
const_local_iterator
cend(size_type __n) const
{ return _M_h.cend(__n); }
float
load_factor() const noexcept
{ return _M_h.load_factor(); }
float
max_load_factor() const noexcept
{ return _M_h.max_load_factor(); }
void
max_load_factor(float __z)
{ _M_h.max_load_factor(__z); }
void
rehash(size_type __n)
{ _M_h.rehash(__n); }
void
reserve(size_type __n)
{ _M_h.reserve(__n); }
template<typename _Key1, typename _Tp1, typename _Hash1, typename _Pred1,
typename _Alloc1>
friend bool
operator==(const unordered_multimap<_Key1, _Tp1,
_Hash1, _Pred1, _Alloc1>&,
const unordered_multimap<_Key1, _Tp1,
_Hash1, _Pred1, _Alloc1>&);
};
template<typename _InputIterator,
typename _Hash = hash<__iter_key_t<_InputIterator>>,
typename _Pred = equal_to<__iter_key_t<_InputIterator>>,
typename _Allocator = allocator<__iter_to_alloc_t<_InputIterator>>,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireNotAllocatorOrIntegral<_Hash>,
typename = _RequireNotAllocator<_Pred>,
typename = _RequireAllocator<_Allocator>>
unordered_multimap(_InputIterator, _InputIterator,
unordered_multimap<int, int>::size_type = {},
_Hash = _Hash(), _Pred = _Pred(),
_Allocator = _Allocator())
-> unordered_multimap<__iter_key_t<_InputIterator>,
__iter_val_t<_InputIterator>, _Hash, _Pred,
_Allocator>;
template<typename _Key, typename _Tp, typename _Hash = hash<_Key>,
typename _Pred = equal_to<_Key>,
typename _Allocator = allocator<pair<const _Key, _Tp>>,
typename = _RequireNotAllocatorOrIntegral<_Hash>,
typename = _RequireNotAllocator<_Pred>,
typename = _RequireAllocator<_Allocator>>
unordered_multimap(initializer_list<pair<_Key, _Tp>>,
unordered_multimap<int, int>::size_type = {},
_Hash = _Hash(), _Pred = _Pred(),
_Allocator = _Allocator())
-> unordered_multimap<_Key, _Tp, _Hash, _Pred, _Allocator>;
template<typename _InputIterator, typename _Allocator,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireAllocator<_Allocator>>
unordered_multimap(_InputIterator, _InputIterator,
unordered_multimap<int, int>::size_type, _Allocator)
-> unordered_multimap<__iter_key_t<_InputIterator>,
__iter_val_t<_InputIterator>,
hash<__iter_key_t<_InputIterator>>,
equal_to<__iter_key_t<_InputIterator>>, _Allocator>;
template<typename _InputIterator, typename _Allocator,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireAllocator<_Allocator>>
unordered_multimap(_InputIterator, _InputIterator, _Allocator)
-> unordered_multimap<__iter_key_t<_InputIterator>,
__iter_val_t<_InputIterator>,
hash<__iter_key_t<_InputIterator>>,
equal_to<__iter_key_t<_InputIterator>>, _Allocator>;
template<typename _InputIterator, typename _Hash, typename _Allocator,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireNotAllocatorOrIntegral<_Hash>,
typename = _RequireAllocator<_Allocator>>
unordered_multimap(_InputIterator, _InputIterator,
unordered_multimap<int, int>::size_type, _Hash,
_Allocator)
-> unordered_multimap<__iter_key_t<_InputIterator>,
__iter_val_t<_InputIterator>, _Hash,
equal_to<__iter_key_t<_InputIterator>>, _Allocator>;
template<typename _Key, typename _Tp, typename _Allocator,
typename = _RequireAllocator<_Allocator>>
unordered_multimap(initializer_list<pair<_Key, _Tp>>,
unordered_multimap<int, int>::size_type,
_Allocator)
-> unordered_multimap<_Key, _Tp, hash<_Key>, equal_to<_Key>, _Allocator>;
template<typename _Key, typename _Tp, typename _Allocator,
typename = _RequireAllocator<_Allocator>>
unordered_multimap(initializer_list<pair<_Key, _Tp>>, _Allocator)
-> unordered_multimap<_Key, _Tp, hash<_Key>, equal_to<_Key>, _Allocator>;
template<typename _Key, typename _Tp, typename _Hash, typename _Allocator,
typename = _RequireNotAllocatorOrIntegral<_Hash>,
typename = _RequireAllocator<_Allocator>>
unordered_multimap(initializer_list<pair<_Key, _Tp>>,
unordered_multimap<int, int>::size_type,
_Hash, _Allocator)
-> unordered_multimap<_Key, _Tp, _Hash, equal_to<_Key>, _Allocator>;
template<class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
inline void
swap(unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __y)
noexcept(noexcept(__x.swap(__y)))
{ __x.swap(__y); }
template<class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
inline void
swap(unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __y)
noexcept(noexcept(__x.swap(__y)))
{ __x.swap(__y); }
template<class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
inline bool
operator==(const unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
const unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __y)
{ return __x._M_h._M_equal(__y._M_h); }
template<class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
inline bool
operator!=(const unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
const unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __y)
{ return !(__x == __y); }
template<class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
inline bool
operator==(const unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
const unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __y)
{ return __x._M_h._M_equal(__y._M_h); }
template<class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
inline bool
operator!=(const unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
const unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __y)
{ return !(__x == __y); }
template<typename _Key, typename _Val, typename _Hash1, typename _Eq1,
typename _Alloc, typename _Hash2, typename _Eq2>
struct _Hash_merge_helper<
std::unordered_map<_Key, _Val, _Hash1, _Eq1, _Alloc>,
_Hash2, _Eq2>
{
private:
template<typename... _Tp>
using unordered_map = std::unordered_map<_Tp...>;
template<typename... _Tp>
using unordered_multimap = std::unordered_multimap<_Tp...>;
friend unordered_map<_Key, _Val, _Hash1, _Eq1, _Alloc>;
static auto&
_S_get_table(unordered_map<_Key, _Val, _Hash2, _Eq2, _Alloc>& __map)
{ return __map._M_h; }
static auto&
_S_get_table(unordered_multimap<_Key, _Val, _Hash2, _Eq2, _Alloc>& __map)
{ return __map._M_h; }
};
template<typename _Key, typename _Val, typename _Hash1, typename _Eq1,
typename _Alloc, typename _Hash2, typename _Eq2>
struct _Hash_merge_helper<
std::unordered_multimap<_Key, _Val, _Hash1, _Eq1, _Alloc>,
_Hash2, _Eq2>
{
private:
template<typename... _Tp>
using unordered_map = std::unordered_map<_Tp...>;
template<typename... _Tp>
using unordered_multimap = std::unordered_multimap<_Tp...>;
friend unordered_multimap<_Key, _Val, _Hash1, _Eq1, _Alloc>;
static auto&
_S_get_table(unordered_map<_Key, _Val, _Hash2, _Eq2, _Alloc>& __map)
{ return __map._M_h; }
static auto&
_S_get_table(unordered_multimap<_Key, _Val, _Hash2, _Eq2, _Alloc>& __map)
{ return __map._M_h; }
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace pmr
{
template<typename _Key, typename _Tp, typename _Hash = std::hash<_Key>,
typename _Pred = std::equal_to<_Key>>
using unordered_map
= std::unordered_map<_Key, _Tp, _Hash, _Pred,
polymorphic_allocator<pair<const _Key, _Tp>>>;
template<typename _Key, typename _Tp, typename _Hash = std::hash<_Key>,
typename _Pred = std::equal_to<_Key>>
using unordered_multimap
= std::unordered_multimap<_Key, _Tp, _Hash, _Pred,
polymorphic_allocator<pair<const _Key, _Tp>>>;
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Tp, size_t _Nm>
struct __array_traits
{
using _Type = _Tp[_Nm];
using _Is_swappable = __is_swappable<_Tp>;
using _Is_nothrow_swappable = __is_nothrow_swappable<_Tp>;
};
template<typename _Tp>
struct __array_traits<_Tp, 0>
{
struct _Type
{
__attribute__((__always_inline__,__noreturn__))
_Tp& operator[](size_t) const noexcept { __builtin_trap(); }
__attribute__((__always_inline__))
constexpr explicit operator _Tp*() const noexcept { return nullptr; }
};
using _Is_swappable = true_type;
using _Is_nothrow_swappable = true_type;
};
template<typename _Tp, std::size_t _Nm>
struct array
{
typedef _Tp value_type;
typedef value_type* pointer;
typedef const value_type* const_pointer;
typedef value_type& reference;
typedef const value_type& const_reference;
typedef value_type* iterator;
typedef const value_type* const_iterator;
typedef std::size_t size_type;
typedef std::ptrdiff_t difference_type;
typedef std::reverse_iterator<iterator> reverse_iterator;
typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
typename __array_traits<_Tp, _Nm>::_Type _M_elems;
void
fill(const value_type& __u)
{ std::fill_n(begin(), size(), __u); }
void
swap(array& __other)
noexcept(__array_traits<_Tp, _Nm>::_Is_nothrow_swappable::value)
{ std::swap_ranges(begin(), end(), __other.begin()); }
[[__gnu__::__const__, __nodiscard__]]
constexpr iterator
begin() noexcept
{ return iterator(data()); }
[[__nodiscard__]]
constexpr const_iterator
begin() const noexcept
{ return const_iterator(data()); }
[[__gnu__::__const__, __nodiscard__]]
constexpr iterator
end() noexcept
{ return iterator(data() + _Nm); }
[[__nodiscard__]]
constexpr const_iterator
end() const noexcept
{ return const_iterator(data() + _Nm); }
[[__gnu__::__const__, __nodiscard__]]
constexpr reverse_iterator
rbegin() noexcept
{ return reverse_iterator(end()); }
[[__nodiscard__]]
constexpr const_reverse_iterator
rbegin() const noexcept
{ return const_reverse_iterator(end()); }
[[__gnu__::__const__, __nodiscard__]]
constexpr reverse_iterator
rend() noexcept
{ return reverse_iterator(begin()); }
[[__nodiscard__]]
constexpr const_reverse_iterator
rend() const noexcept
{ return const_reverse_iterator(begin()); }
[[__nodiscard__]]
constexpr const_iterator
cbegin() const noexcept
{ return const_iterator(data()); }
[[__nodiscard__]]
constexpr const_iterator
cend() const noexcept
{ return const_iterator(data() + _Nm); }
[[__nodiscard__]]
constexpr const_reverse_iterator
crbegin() const noexcept
{ return const_reverse_iterator(end()); }
[[__nodiscard__]]
constexpr const_reverse_iterator
crend() const noexcept
{ return const_reverse_iterator(begin()); }
[[__nodiscard__, __gnu__::__const__, __gnu__::__always_inline__]]
constexpr size_type
size() const noexcept { return _Nm; }
[[__nodiscard__, __gnu__::__const__, __gnu__::__always_inline__]]
constexpr size_type
max_size() const noexcept { return _Nm; }
[[__nodiscard__, __gnu__::__const__, __gnu__::__always_inline__]]
constexpr bool
empty() const noexcept { return size() == 0; }
[[__nodiscard__]]
constexpr reference
operator[](size_type __n) noexcept
{
;
return _M_elems[__n];
}
[[__nodiscard__]]
constexpr const_reference
operator[](size_type __n) const noexcept
{
;
return _M_elems[__n];
}
constexpr reference
at(size_type __n)
{
if (__n >= _Nm)
std::__throw_out_of_range_fmt(("array::at: __n (which is %zu) " ">= _Nm (which is %zu)")
,
__n, _Nm);
return _M_elems[__n];
}
constexpr const_reference
at(size_type __n) const
{
return __n < _Nm ? _M_elems[__n]
: (std::__throw_out_of_range_fmt(("array::at: __n (which is %zu) " ">= _Nm (which is %zu)")
,
__n, _Nm),
_M_elems[__n]);
}
[[__nodiscard__]]
constexpr reference
front() noexcept
{
;
return _M_elems[(size_type)0];
}
[[__nodiscard__]]
constexpr const_reference
front() const noexcept
{
;
return _M_elems[(size_type)0];
}
[[__nodiscard__]]
constexpr reference
back() noexcept
{
;
return _M_elems[_Nm - 1];
}
[[__nodiscard__]]
constexpr const_reference
back() const noexcept
{
;
return _M_elems[_Nm - 1];
}
[[__nodiscard__, __gnu__::__const__, __gnu__::__always_inline__]]
constexpr pointer
data() noexcept
{ return static_cast<pointer>(_M_elems); }
[[__nodiscard__]]
constexpr const_pointer
data() const noexcept
{ return static_cast<const_pointer>(_M_elems); }
};
template<typename _Tp, typename... _Up>
array(_Tp, _Up...)
-> array<enable_if_t<(is_same_v<_Tp, _Up> && ...), _Tp>,
1 + sizeof...(_Up)>;
template<typename _Tp, std::size_t _Nm>
[[__nodiscard__]]
inline bool
operator==(const array<_Tp, _Nm>& __one, const array<_Tp, _Nm>& __two)
{ return std::equal(__one.begin(), __one.end(), __two.begin()); }
template<typename _Tp, std::size_t _Nm>
[[__nodiscard__]]
inline bool
operator!=(const array<_Tp, _Nm>& __one, const array<_Tp, _Nm>& __two)
{ return !(__one == __two); }
template<typename _Tp, std::size_t _Nm>
[[__nodiscard__]]
inline bool
operator<(const array<_Tp, _Nm>& __a, const array<_Tp, _Nm>& __b)
{
return std::lexicographical_compare(__a.begin(), __a.end(),
__b.begin(), __b.end());
}
template<typename _Tp, std::size_t _Nm>
[[__nodiscard__]]
inline bool
operator>(const array<_Tp, _Nm>& __one, const array<_Tp, _Nm>& __two)
{ return __two < __one; }
template<typename _Tp, std::size_t _Nm>
[[__nodiscard__]]
inline bool
operator<=(const array<_Tp, _Nm>& __one, const array<_Tp, _Nm>& __two)
{ return !(__one > __two); }
template<typename _Tp, std::size_t _Nm>
[[__nodiscard__]]
inline bool
operator>=(const array<_Tp, _Nm>& __one, const array<_Tp, _Nm>& __two)
{ return !(__one < __two); }
template<typename _Tp, std::size_t _Nm>
inline
__enable_if_t<__array_traits<_Tp, _Nm>::_Is_swappable::value>
swap(array<_Tp, _Nm>& __one, array<_Tp, _Nm>& __two)
noexcept(noexcept(__one.swap(__two)))
{ __one.swap(__two); }
template<typename _Tp, std::size_t _Nm>
__enable_if_t<!__array_traits<_Tp, _Nm>::_Is_swappable::value>
swap(array<_Tp, _Nm>&, array<_Tp, _Nm>&) = delete;
template<std::size_t _Int, typename _Tp, std::size_t _Nm>
[[__nodiscard__]]
constexpr _Tp&
get(array<_Tp, _Nm>& __arr) noexcept
{
static_assert(_Int < _Nm, "array index is within bounds");
return __arr._M_elems[_Int];
}
template<std::size_t _Int, typename _Tp, std::size_t _Nm>
[[__nodiscard__]]
constexpr _Tp&&
get(array<_Tp, _Nm>&& __arr) noexcept
{
static_assert(_Int < _Nm, "array index is within bounds");
return std::move(std::get<_Int>(__arr));
}
template<std::size_t _Int, typename _Tp, std::size_t _Nm>
[[__nodiscard__]]
constexpr const _Tp&
get(const array<_Tp, _Nm>& __arr) noexcept
{
static_assert(_Int < _Nm, "array index is within bounds");
return __arr._M_elems[_Int];
}
template<std::size_t _Int, typename _Tp, std::size_t _Nm>
[[__nodiscard__]]
constexpr const _Tp&&
get(const array<_Tp, _Nm>&& __arr) noexcept
{
static_assert(_Int < _Nm, "array index is within bounds");
return std::move(std::get<_Int>(__arr));
}
template<typename _Tp, size_t _Nm>
struct tuple_size<array<_Tp, _Nm>>
: public integral_constant<size_t, _Nm> { };
template<size_t _Ind, typename _Tp, size_t _Nm>
struct tuple_element<_Ind, array<_Tp, _Nm>>
{
static_assert(_Ind < _Nm, "array index is in range");
using type = _Tp;
};
template<typename _Tp, size_t _Nm>
inline constexpr size_t tuple_size_v<array<_Tp, _Nm>> = _Nm;
template<typename _Tp, size_t _Nm>
inline constexpr size_t tuple_size_v<const array<_Tp, _Nm>> = _Nm;
template<typename _Tp, size_t _Nm>
struct __is_tuple_like_impl<array<_Tp, _Nm>> : true_type
{ };
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _IIter, typename _Predicate>
bool
all_of(_IIter, _IIter, _Predicate);
template<typename _IIter, typename _Predicate>
bool
any_of(_IIter, _IIter, _Predicate);
template<typename _FIter, typename _Tp>
bool
binary_search(_FIter, _FIter, const _Tp&);
template<typename _FIter, typename _Tp, typename _Compare>
bool
binary_search(_FIter, _FIter, const _Tp&, _Compare);
template<typename _Tp>
constexpr
const _Tp&
clamp(const _Tp&, const _Tp&, const _Tp&);
template<typename _Tp, typename _Compare>
constexpr
const _Tp&
clamp(const _Tp&, const _Tp&, const _Tp&, _Compare);
template<typename _IIter, typename _OIter>
_OIter
copy(_IIter, _IIter, _OIter);
template<typename _BIter1, typename _BIter2>
_BIter2
copy_backward(_BIter1, _BIter1, _BIter2);
template<typename _IIter, typename _OIter, typename _Predicate>
_OIter
copy_if(_IIter, _IIter, _OIter, _Predicate);
template<typename _IIter, typename _Size, typename _OIter>
_OIter
copy_n(_IIter, _Size, _OIter);
template<typename _FIter, typename _Tp>
pair<_FIter, _FIter>
equal_range(_FIter, _FIter, const _Tp&);
template<typename _FIter, typename _Tp, typename _Compare>
pair<_FIter, _FIter>
equal_range(_FIter, _FIter, const _Tp&, _Compare);
template<typename _FIter, typename _Tp>
void
fill(_FIter, _FIter, const _Tp&);
template<typename _OIter, typename _Size, typename _Tp>
_OIter
fill_n(_OIter, _Size, const _Tp&);
template<typename _FIter1, typename _FIter2>
_FIter1
find_end(_FIter1, _FIter1, _FIter2, _FIter2);
template<typename _FIter1, typename _FIter2, typename _BinaryPredicate>
_FIter1
find_end(_FIter1, _FIter1, _FIter2, _FIter2, _BinaryPredicate);
template<typename _IIter, typename _Predicate>
_IIter
find_if_not(_IIter, _IIter, _Predicate);
template<typename _IIter1, typename _IIter2>
bool
includes(_IIter1, _IIter1, _IIter2, _IIter2);
template<typename _IIter1, typename _IIter2, typename _Compare>
bool
includes(_IIter1, _IIter1, _IIter2, _IIter2, _Compare);
template<typename _BIter>
void
inplace_merge(_BIter, _BIter, _BIter);
template<typename _BIter, typename _Compare>
void
inplace_merge(_BIter, _BIter, _BIter, _Compare);
template<typename _RAIter>
bool
is_heap(_RAIter, _RAIter);
template<typename _RAIter, typename _Compare>
bool
is_heap(_RAIter, _RAIter, _Compare);
template<typename _RAIter>
_RAIter
is_heap_until(_RAIter, _RAIter);
template<typename _RAIter, typename _Compare>
_RAIter
is_heap_until(_RAIter, _RAIter, _Compare);
template<typename _IIter, typename _Predicate>
bool
is_partitioned(_IIter, _IIter, _Predicate);
template<typename _FIter1, typename _FIter2>
bool
is_permutation(_FIter1, _FIter1, _FIter2);
template<typename _FIter1, typename _FIter2,
typename _BinaryPredicate>
bool
is_permutation(_FIter1, _FIter1, _FIter2, _BinaryPredicate);
template<typename _FIter>
bool
is_sorted(_FIter, _FIter);
template<typename _FIter, typename _Compare>
bool
is_sorted(_FIter, _FIter, _Compare);
template<typename _FIter>
_FIter
is_sorted_until(_FIter, _FIter);
template<typename _FIter, typename _Compare>
_FIter
is_sorted_until(_FIter, _FIter, _Compare);
template<typename _FIter1, typename _FIter2>
void
iter_swap(_FIter1, _FIter2);
template<typename _FIter, typename _Tp>
_FIter
lower_bound(_FIter, _FIter, const _Tp&);
template<typename _FIter, typename _Tp, typename _Compare>
_FIter
lower_bound(_FIter, _FIter, const _Tp&, _Compare);
template<typename _RAIter>
void
make_heap(_RAIter, _RAIter);
template<typename _RAIter, typename _Compare>
void
make_heap(_RAIter, _RAIter, _Compare);
template<typename _Tp>
constexpr
const _Tp&
max(const _Tp&, const _Tp&);
template<typename _Tp, typename _Compare>
constexpr
const _Tp&
max(const _Tp&, const _Tp&, _Compare);
template<typename _Tp>
constexpr
const _Tp&
min(const _Tp&, const _Tp&);
template<typename _Tp, typename _Compare>
constexpr
const _Tp&
min(const _Tp&, const _Tp&, _Compare);
template<typename _Tp>
constexpr
pair<const _Tp&, const _Tp&>
minmax(const _Tp&, const _Tp&);
template<typename _Tp, typename _Compare>
constexpr
pair<const _Tp&, const _Tp&>
minmax(const _Tp&, const _Tp&, _Compare);
template<typename _FIter>
constexpr
pair<_FIter, _FIter>
minmax_element(_FIter, _FIter);
template<typename _FIter, typename _Compare>
constexpr
pair<_FIter, _FIter>
minmax_element(_FIter, _FIter, _Compare);
template<typename _Tp>
constexpr
_Tp
min(initializer_list<_Tp>);
template<typename _Tp, typename _Compare>
constexpr
_Tp
min(initializer_list<_Tp>, _Compare);
template<typename _Tp>
constexpr
_Tp
max(initializer_list<_Tp>);
template<typename _Tp, typename _Compare>
constexpr
_Tp
max(initializer_list<_Tp>, _Compare);
template<typename _Tp>
constexpr
pair<_Tp, _Tp>
minmax(initializer_list<_Tp>);
template<typename _Tp, typename _Compare>
constexpr
pair<_Tp, _Tp>
minmax(initializer_list<_Tp>, _Compare);
template<typename _BIter>
bool
next_permutation(_BIter, _BIter);
template<typename _BIter, typename _Compare>
bool
next_permutation(_BIter, _BIter, _Compare);
template<typename _IIter, typename _Predicate>
bool
none_of(_IIter, _IIter, _Predicate);
template<typename _IIter, typename _RAIter>
_RAIter
partial_sort_copy(_IIter, _IIter, _RAIter, _RAIter);
template<typename _IIter, typename _RAIter, typename _Compare>
_RAIter
partial_sort_copy(_IIter, _IIter, _RAIter, _RAIter, _Compare);
template<typename _IIter, typename _OIter1,
typename _OIter2, typename _Predicate>
pair<_OIter1, _OIter2>
partition_copy(_IIter, _IIter, _OIter1, _OIter2, _Predicate);
template<typename _FIter, typename _Predicate>
_FIter
partition_point(_FIter, _FIter, _Predicate);
template<typename _RAIter>
void
pop_heap(_RAIter, _RAIter);
template<typename _RAIter, typename _Compare>
void
pop_heap(_RAIter, _RAIter, _Compare);
template<typename _BIter>
bool
prev_permutation(_BIter, _BIter);
template<typename _BIter, typename _Compare>
bool
prev_permutation(_BIter, _BIter, _Compare);
template<typename _RAIter>
void
push_heap(_RAIter, _RAIter);
template<typename _RAIter, typename _Compare>
void
push_heap(_RAIter, _RAIter, _Compare);
template<typename _FIter, typename _Tp>
_FIter
remove(_FIter, _FIter, const _Tp&);
template<typename _FIter, typename _Predicate>
_FIter
remove_if(_FIter, _FIter, _Predicate);
template<typename _IIter, typename _OIter, typename _Tp>
_OIter
remove_copy(_IIter, _IIter, _OIter, const _Tp&);
template<typename _IIter, typename _OIter, typename _Predicate>
_OIter
remove_copy_if(_IIter, _IIter, _OIter, _Predicate);
template<typename _IIter, typename _OIter, typename _Tp>
_OIter
replace_copy(_IIter, _IIter, _OIter, const _Tp&, const _Tp&);
template<typename _Iter, typename _OIter, typename _Predicate, typename _Tp>
_OIter
replace_copy_if(_Iter, _Iter, _OIter, _Predicate, const _Tp&);
template<typename _BIter>
void
reverse(_BIter, _BIter);
template<typename _BIter, typename _OIter>
_OIter
reverse_copy(_BIter, _BIter, _OIter);
inline namespace _V2 {
template<typename _FIter>
_FIter
rotate(_FIter, _FIter, _FIter);
}
template<typename _FIter, typename _OIter>
_OIter
rotate_copy(_FIter, _FIter, _FIter, _OIter);
template<typename _RAIter, typename _UGenerator>
void
shuffle(_RAIter, _RAIter, _UGenerator&&);
template<typename _RAIter>
void
sort_heap(_RAIter, _RAIter);
template<typename _RAIter, typename _Compare>
void
sort_heap(_RAIter, _RAIter, _Compare);
template<typename _BIter, typename _Predicate>
_BIter
stable_partition(_BIter, _BIter, _Predicate);
template<typename _FIter1, typename _FIter2>
_FIter2
swap_ranges(_FIter1, _FIter1, _FIter2);
template<typename _FIter>
_FIter
unique(_FIter, _FIter);
template<typename _FIter, typename _BinaryPredicate>
_FIter
unique(_FIter, _FIter, _BinaryPredicate);
template<typename _FIter, typename _Tp>
_FIter
upper_bound(_FIter, _FIter, const _Tp&);
template<typename _FIter, typename _Tp, typename _Compare>
_FIter
upper_bound(_FIter, _FIter, const _Tp&, _Compare);
template<typename _FIter>
_FIter
adjacent_find(_FIter, _FIter);
template<typename _FIter, typename _BinaryPredicate>
_FIter
adjacent_find(_FIter, _FIter, _BinaryPredicate);
template<typename _IIter, typename _Tp>
typename iterator_traits<_IIter>::difference_type
count(_IIter, _IIter, const _Tp&);
template<typename _IIter, typename _Predicate>
typename iterator_traits<_IIter>::difference_type
count_if(_IIter, _IIter, _Predicate);
template<typename _IIter1, typename _IIter2>
bool
equal(_IIter1, _IIter1, _IIter2);
template<typename _IIter1, typename _IIter2, typename _BinaryPredicate>
bool
equal(_IIter1, _IIter1, _IIter2, _BinaryPredicate);
template<typename _IIter, typename _Tp>
_IIter
find(_IIter, _IIter, const _Tp&);
template<typename _FIter1, typename _FIter2>
_FIter1
find_first_of(_FIter1, _FIter1, _FIter2, _FIter2);
template<typename _FIter1, typename _FIter2, typename _BinaryPredicate>
_FIter1
find_first_of(_FIter1, _FIter1, _FIter2, _FIter2, _BinaryPredicate);
template<typename _IIter, typename _Predicate>
_IIter
find_if(_IIter, _IIter, _Predicate);
template<typename _IIter, typename _Funct>
_Funct
for_each(_IIter, _IIter, _Funct);
template<typename _FIter, typename _Generator>
void
generate(_FIter, _FIter, _Generator);
template<typename _OIter, typename _Size, typename _Generator>
_OIter
generate_n(_OIter, _Size, _Generator);
template<typename _IIter1, typename _IIter2>
bool
lexicographical_compare(_IIter1, _IIter1, _IIter2, _IIter2);
template<typename _IIter1, typename _IIter2, typename _Compare>
bool
lexicographical_compare(_IIter1, _IIter1, _IIter2, _IIter2, _Compare);
template<typename _FIter>
constexpr
_FIter
max_element(_FIter, _FIter);
template<typename _FIter, typename _Compare>
constexpr
_FIter
max_element(_FIter, _FIter, _Compare);
template<typename _IIter1, typename _IIter2, typename _OIter>
_OIter
merge(_IIter1, _IIter1, _IIter2, _IIter2, _OIter);
template<typename _IIter1, typename _IIter2, typename _OIter,
typename _Compare>
_OIter
merge(_IIter1, _IIter1, _IIter2, _IIter2, _OIter, _Compare);
template<typename _FIter>
constexpr
_FIter
min_element(_FIter, _FIter);
template<typename _FIter, typename _Compare>
constexpr
_FIter
min_element(_FIter, _FIter, _Compare);
template<typename _IIter1, typename _IIter2>
pair<_IIter1, _IIter2>
mismatch(_IIter1, _IIter1, _IIter2);
template<typename _IIter1, typename _IIter2, typename _BinaryPredicate>
pair<_IIter1, _IIter2>
mismatch(_IIter1, _IIter1, _IIter2, _BinaryPredicate);
template<typename _RAIter>
void
nth_element(_RAIter, _RAIter, _RAIter);
template<typename _RAIter, typename _Compare>
void
nth_element(_RAIter, _RAIter, _RAIter, _Compare);
template<typename _RAIter>
void
partial_sort(_RAIter, _RAIter, _RAIter);
template<typename _RAIter, typename _Compare>
void
partial_sort(_RAIter, _RAIter, _RAIter, _Compare);
template<typename _BIter, typename _Predicate>
_BIter
partition(_BIter, _BIter, _Predicate);
template<typename _RAIter>
__attribute__ ((__deprecated__ ("use '" "std::shuffle" "' instead")))
void
random_shuffle(_RAIter, _RAIter);
template<typename _RAIter, typename _Generator>
__attribute__ ((__deprecated__ ("use '" "std::shuffle" "' instead")))
void
random_shuffle(_RAIter, _RAIter,
_Generator&&);
template<typename _FIter, typename _Tp>
void
replace(_FIter, _FIter, const _Tp&, const _Tp&);
template<typename _FIter, typename _Predicate, typename _Tp>
void
replace_if(_FIter, _FIter, _Predicate, const _Tp&);
template<typename _FIter1, typename _FIter2>
_FIter1
search(_FIter1, _FIter1, _FIter2, _FIter2);
template<typename _FIter1, typename _FIter2, typename _BinaryPredicate>
_FIter1
search(_FIter1, _FIter1, _FIter2, _FIter2, _BinaryPredicate);
template<typename _FIter, typename _Size, typename _Tp>
_FIter
search_n(_FIter, _FIter, _Size, const _Tp&);
template<typename _FIter, typename _Size, typename _Tp,
typename _BinaryPredicate>
_FIter
search_n(_FIter, _FIter, _Size, const _Tp&, _BinaryPredicate);
template<typename _IIter1, typename _IIter2, typename _OIter>
_OIter
set_difference(_IIter1, _IIter1, _IIter2, _IIter2, _OIter);
template<typename _IIter1, typename _IIter2, typename _OIter,
typename _Compare>
_OIter
set_difference(_IIter1, _IIter1, _IIter2, _IIter2, _OIter, _Compare);
template<typename _IIter1, typename _IIter2, typename _OIter>
_OIter
set_intersection(_IIter1, _IIter1, _IIter2, _IIter2, _OIter);
template<typename _IIter1, typename _IIter2, typename _OIter,
typename _Compare>
_OIter
set_intersection(_IIter1, _IIter1, _IIter2, _IIter2, _OIter, _Compare);
template<typename _IIter1, typename _IIter2, typename _OIter>
_OIter
set_symmetric_difference(_IIter1, _IIter1, _IIter2, _IIter2, _OIter);
template<typename _IIter1, typename _IIter2, typename _OIter,
typename _Compare>
_OIter
set_symmetric_difference(_IIter1, _IIter1, _IIter2, _IIter2,
_OIter, _Compare);
template<typename _IIter1, typename _IIter2, typename _OIter>
_OIter
set_union(_IIter1, _IIter1, _IIter2, _IIter2, _OIter);
template<typename _IIter1, typename _IIter2, typename _OIter,
typename _Compare>
_OIter
set_union(_IIter1, _IIter1, _IIter2, _IIter2, _OIter, _Compare);
template<typename _RAIter>
void
sort(_RAIter, _RAIter);
template<typename _RAIter, typename _Compare>
void
sort(_RAIter, _RAIter, _Compare);
template<typename _RAIter>
void
stable_sort(_RAIter, _RAIter);
template<typename _RAIter, typename _Compare>
void
stable_sort(_RAIter, _RAIter, _Compare);
template<typename _IIter, typename _OIter, typename _UnaryOperation>
_OIter
transform(_IIter, _IIter, _OIter, _UnaryOperation);
template<typename _IIter1, typename _IIter2, typename _OIter,
typename _BinaryOperation>
_OIter
transform(_IIter1, _IIter1, _IIter2, _OIter, _BinaryOperation);
template<typename _IIter, typename _OIter>
_OIter
unique_copy(_IIter, _IIter, _OIter);
template<typename _IIter, typename _OIter, typename _BinaryPredicate>
_OIter
unique_copy(_IIter, _IIter, _OIter, _BinaryPredicate);
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _RandomAccessIterator, typename _Distance,
typename _Compare>
_Distance
__is_heap_until(_RandomAccessIterator __first, _Distance __n,
_Compare& __comp)
{
_Distance __parent = 0;
for (_Distance __child = 1; __child < __n; ++__child)
{
if (__comp(__first + __parent, __first + __child))
return __child;
if ((__child & 1) == 0)
++__parent;
}
return __n;
}
template<typename _RandomAccessIterator, typename _Distance>
inline bool
__is_heap(_RandomAccessIterator __first, _Distance __n)
{
__gnu_cxx::__ops::_Iter_less_iter __comp;
return std::__is_heap_until(__first, __n, __comp) == __n;
}
template<typename _RandomAccessIterator, typename _Compare,
typename _Distance>
inline bool
__is_heap(_RandomAccessIterator __first, _Compare __comp, _Distance __n)
{
typedef __decltype(__comp) _Cmp;
__gnu_cxx::__ops::_Iter_comp_iter<_Cmp> __cmp(std::move(__comp));
return std::__is_heap_until(__first, __n, __cmp) == __n;
}
template<typename _RandomAccessIterator>
inline bool
__is_heap(_RandomAccessIterator __first, _RandomAccessIterator __last)
{ return std::__is_heap(__first, std::distance(__first, __last)); }
template<typename _RandomAccessIterator, typename _Compare>
inline bool
__is_heap(_RandomAccessIterator __first, _RandomAccessIterator __last,
_Compare __comp)
{
return std::__is_heap(__first, std::move(__comp),
std::distance(__first, __last));
}
template<typename _RandomAccessIterator, typename _Distance, typename _Tp,
typename _Compare>
void
__push_heap(_RandomAccessIterator __first,
_Distance __holeIndex, _Distance __topIndex, _Tp __value,
_Compare& __comp)
{
_Distance __parent = (__holeIndex - 1) / 2;
while (__holeIndex > __topIndex && __comp(__first + __parent, __value))
{
*(__first + __holeIndex) = std::move(*(__first + __parent));
__holeIndex = __parent;
__parent = (__holeIndex - 1) / 2;
}
*(__first + __holeIndex) = std::move(__value);
}
template<typename _RandomAccessIterator>
inline void
push_heap(_RandomAccessIterator __first, _RandomAccessIterator __last)
{
typedef typename iterator_traits<_RandomAccessIterator>::value_type
_ValueType;
typedef typename iterator_traits<_RandomAccessIterator>::difference_type
_DistanceType;
;
;
;
__gnu_cxx::__ops::_Iter_less_val __comp;
_ValueType __value = std::move(*(__last - 1));
std::__push_heap(__first, _DistanceType((__last - __first) - 1),
_DistanceType(0), std::move(__value), __comp);
}
template<typename _RandomAccessIterator, typename _Compare>
inline void
push_heap(_RandomAccessIterator __first, _RandomAccessIterator __last,
_Compare __comp)
{
typedef typename iterator_traits<_RandomAccessIterator>::value_type
_ValueType;
typedef typename iterator_traits<_RandomAccessIterator>::difference_type
_DistanceType;
;
;
;
__decltype(__gnu_cxx::__ops::__iter_comp_val(std::move(__comp)))
__cmp(std::move(__comp));
_ValueType __value = std::move(*(__last - 1));
std::__push_heap(__first, _DistanceType((__last - __first) - 1),
_DistanceType(0), std::move(__value), __cmp);
}
template<typename _RandomAccessIterator, typename _Distance,
typename _Tp, typename _Compare>
void
__adjust_heap(_RandomAccessIterator __first, _Distance __holeIndex,
_Distance __len, _Tp __value, _Compare __comp)
{
const _Distance __topIndex = __holeIndex;
_Distance __secondChild = __holeIndex;
while (__secondChild < (__len - 1) / 2)
{
__secondChild = 2 * (__secondChild + 1);
if (__comp(__first + __secondChild,
__first + (__secondChild - 1)))
__secondChild--;
*(__first + __holeIndex) = std::move(*(__first + __secondChild));
__holeIndex = __secondChild;
}
if ((__len & 1) == 0 && __secondChild == (__len - 2) / 2)
{
__secondChild = 2 * (__secondChild + 1);
*(__first + __holeIndex) = std::move(*(__first + (__secondChild - 1)))
;
__holeIndex = __secondChild - 1;
}
__decltype(__gnu_cxx::__ops::__iter_comp_val(std::move(__comp)))
__cmp(std::move(__comp));
std::__push_heap(__first, __holeIndex, __topIndex,
std::move(__value), __cmp);
}
template<typename _RandomAccessIterator, typename _Compare>
inline void
__pop_heap(_RandomAccessIterator __first, _RandomAccessIterator __last,
_RandomAccessIterator __result, _Compare& __comp)
{
typedef typename iterator_traits<_RandomAccessIterator>::value_type
_ValueType;
typedef typename iterator_traits<_RandomAccessIterator>::difference_type
_DistanceType;
_ValueType __value = std::move(*__result);
*__result = std::move(*__first);
std::__adjust_heap(__first, _DistanceType(0),
_DistanceType(__last - __first),
std::move(__value), __comp);
}
template<typename _RandomAccessIterator>
inline void
pop_heap(_RandomAccessIterator __first, _RandomAccessIterator __last)
{
;
;
;
;
if (__last - __first > 1)
{
--__last;
__gnu_cxx::__ops::_Iter_less_iter __comp;
std::__pop_heap(__first, __last, __last, __comp);
}
}
template<typename _RandomAccessIterator, typename _Compare>
inline void
pop_heap(_RandomAccessIterator __first,
_RandomAccessIterator __last, _Compare __comp)
{
;
;
;
;
if (__last - __first > 1)
{
typedef __decltype(__comp) _Cmp;
__gnu_cxx::__ops::_Iter_comp_iter<_Cmp> __cmp(std::move(__comp));
--__last;
std::__pop_heap(__first, __last, __last, __cmp);
}
}
template<typename _RandomAccessIterator, typename _Compare>
void
__make_heap(_RandomAccessIterator __first, _RandomAccessIterator __last,
_Compare& __comp)
{
typedef typename iterator_traits<_RandomAccessIterator>::value_type
_ValueType;
typedef typename iterator_traits<_RandomAccessIterator>::difference_type
_DistanceType;
if (__last - __first < 2)
return;
const _DistanceType __len = __last - __first;
_DistanceType __parent = (__len - 2) / 2;
while (true)
{
_ValueType __value = std::move(*(__first + __parent));
std::__adjust_heap(__first, __parent, __len, std::move(__value),
__comp);
if (__parent == 0)
return;
__parent--;
}
}
template<typename _RandomAccessIterator>
inline void
make_heap(_RandomAccessIterator __first, _RandomAccessIterator __last)
{
;
;
__gnu_cxx::__ops::_Iter_less_iter __comp;
std::__make_heap(__first, __last, __comp);
}
template<typename _RandomAccessIterator, typename _Compare>
inline void
make_heap(_RandomAccessIterator __first, _RandomAccessIterator __last,
_Compare __comp)
{
;
;
typedef __decltype(__comp) _Cmp;
__gnu_cxx::__ops::_Iter_comp_iter<_Cmp> __cmp(std::move(__comp));
std::__make_heap(__first, __last, __cmp);
}
template<typename _RandomAccessIterator, typename _Compare>
void
__sort_heap(_RandomAccessIterator __first, _RandomAccessIterator __last,
_Compare& __comp)
{
while (__last - __first > 1)
{
--__last;
std::__pop_heap(__first, __last, __last, __comp);
}
}
template<typename _RandomAccessIterator>
inline void
sort_heap(_RandomAccessIterator __first, _RandomAccessIterator __last)
{
;
;
;
__gnu_cxx::__ops::_Iter_less_iter __comp;
std::__sort_heap(__first, __last, __comp);
}
template<typename _RandomAccessIterator, typename _Compare>
inline void
sort_heap(_RandomAccessIterator __first, _RandomAccessIterator __last,
_Compare __comp)
{
;
;
;
typedef __decltype(__comp) _Cmp;
__gnu_cxx::__ops::_Iter_comp_iter<_Cmp> __cmp(std::move(__comp));
std::__sort_heap(__first, __last, __cmp);
}
template<typename _RandomAccessIterator>
inline _RandomAccessIterator
is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last)
{
;
;
__gnu_cxx::__ops::_Iter_less_iter __comp;
return __first +
std::__is_heap_until(__first, std::distance(__first, __last), __comp);
}
template<typename _RandomAccessIterator, typename _Compare>
inline _RandomAccessIterator
is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last,
_Compare __comp)
{
;
;
typedef __decltype(__comp) _Cmp;
__gnu_cxx::__ops::_Iter_comp_iter<_Cmp> __cmp(std::move(__comp));
return __first
+ std::__is_heap_until(__first, std::distance(__first, __last), __cmp);
}
template<typename _RandomAccessIterator>
inline bool
is_heap(_RandomAccessIterator __first, _RandomAccessIterator __last)
{ return std::is_heap_until(__first, __last) == __last; }
template<typename _RandomAccessIterator, typename _Compare>
inline bool
is_heap(_RandomAccessIterator __first, _RandomAccessIterator __last,
_Compare __comp)
{
;
;
const auto __dist = std::distance(__first, __last);
typedef __decltype(__comp) _Cmp;
__gnu_cxx::__ops::_Iter_comp_iter<_Cmp> __cmp(std::move(__comp));
return std::__is_heap_until(__first, __dist, __cmp) == __dist;
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace __detail
{
template<typename _Tp>
constexpr bool
_Power_of_2(_Tp __x)
{
return ((__x - 1) & __x) == 0;
}
}
template<typename _IntType = int>
class uniform_int_distribution
{
static_assert(std::is_integral<_IntType>::value,
"template argument must be an integral type");
public:
typedef _IntType result_type;
struct param_type
{
typedef uniform_int_distribution<_IntType> distribution_type;
param_type() : param_type(0) { }
explicit
param_type(_IntType __a,
_IntType __b = __gnu_cxx::__int_traits<_IntType>::__max)
: _M_a(__a), _M_b(__b)
{
do { if (std::__is_constant_evaluated() && !bool(_M_a <= _M_b)) __builtin_unreachable(); } while (false);
}
result_type
a() const
{ return _M_a; }
result_type
b() const
{ return _M_b; }
friend bool
operator==(const param_type& __p1, const param_type& __p2)
{ return __p1._M_a == __p2._M_a && __p1._M_b == __p2._M_b; }
friend bool
operator!=(const param_type& __p1, const param_type& __p2)
{ return !(__p1 == __p2); }
private:
_IntType _M_a;
_IntType _M_b;
};
public:
uniform_int_distribution() : uniform_int_distribution(0) { }
explicit
uniform_int_distribution(_IntType __a,
_IntType __b
= __gnu_cxx::__int_traits<_IntType>::__max)
: _M_param(__a, __b)
{ }
explicit
uniform_int_distribution(const param_type& __p)
: _M_param(__p)
{ }
void
reset() { }
result_type
a() const
{ return _M_param.a(); }
result_type
b() const
{ return _M_param.b(); }
param_type
param() const
{ return _M_param; }
void
param(const param_type& __param)
{ _M_param = __param; }
result_type
min() const
{ return this->a(); }
result_type
max() const
{ return this->b(); }
template<typename _UniformRandomBitGenerator>
result_type
operator()(_UniformRandomBitGenerator& __urng)
{ return this->operator()(__urng, _M_param); }
template<typename _UniformRandomBitGenerator>
result_type
operator()(_UniformRandomBitGenerator& __urng,
const param_type& __p);
template<typename _ForwardIterator,
typename _UniformRandomBitGenerator>
void
__generate(_ForwardIterator __f, _ForwardIterator __t,
_UniformRandomBitGenerator& __urng)
{ this->__generate(__f, __t, __urng, _M_param); }
template<typename _ForwardIterator,
typename _UniformRandomBitGenerator>
void
__generate(_ForwardIterator __f, _ForwardIterator __t,
_UniformRandomBitGenerator& __urng,
const param_type& __p)
{ this->__generate_impl(__f, __t, __urng, __p); }
template<typename _UniformRandomBitGenerator>
void
__generate(result_type* __f, result_type* __t,
_UniformRandomBitGenerator& __urng,
const param_type& __p)
{ this->__generate_impl(__f, __t, __urng, __p); }
friend bool
operator==(const uniform_int_distribution& __d1,
const uniform_int_distribution& __d2)
{ return __d1._M_param == __d2._M_param; }
private:
template<typename _ForwardIterator,
typename _UniformRandomBitGenerator>
void
__generate_impl(_ForwardIterator __f, _ForwardIterator __t,
_UniformRandomBitGenerator& __urng,
const param_type& __p);
param_type _M_param;
template<typename _Wp, typename _Urbg, typename _Up>
static _Up
_S_nd(_Urbg& __g, _Up __range)
{
using _Up_traits = __gnu_cxx::__int_traits<_Up>;
using _Wp_traits = __gnu_cxx::__int_traits<_Wp>;
static_assert(!_Up_traits::__is_signed, "U must be unsigned");
static_assert(!_Wp_traits::__is_signed, "W must be unsigned");
static_assert(_Wp_traits::__digits == (2 * _Up_traits::__digits),
"W must be twice as wide as U");
_Wp __product = _Wp(__g()) * _Wp(__range);
_Up __low = _Up(__product);
if (__low < __range)
{
_Up __threshold = -__range % __range;
while (__low < __threshold)
{
__product = _Wp(__g()) * _Wp(__range);
__low = _Up(__product);
}
}
return __product >> _Up_traits::__digits;
}
};
template<typename _IntType>
template<typename _UniformRandomBitGenerator>
typename uniform_int_distribution<_IntType>::result_type
uniform_int_distribution<_IntType>::
operator()(_UniformRandomBitGenerator& __urng,
const param_type& __param)
{
typedef typename _UniformRandomBitGenerator::result_type _Gresult_type;
typedef typename make_unsigned<result_type>::type __utype;
typedef typename common_type<_Gresult_type, __utype>::type __uctype;
constexpr __uctype __urngmin = _UniformRandomBitGenerator::min();
constexpr __uctype __urngmax = _UniformRandomBitGenerator::max();
static_assert( __urngmin < __urngmax,
"Uniform random bit generator must define min() < max()");
constexpr __uctype __urngrange = __urngmax - __urngmin;
const __uctype __urange
= __uctype(__param.b()) - __uctype(__param.a());
__uctype __ret;
if (__urngrange > __urange)
{
const __uctype __uerange = __urange + 1;
if constexpr (__urngrange == 0xffffffffU)
{
unsigned int __u32erange = __uerange;
__ret = _S_nd<long long unsigned int>(__urng, __u32erange);
}
else
{
const __uctype __scaling = __urngrange / __uerange;
const __uctype __past = __uerange * __scaling;
do
__ret = __uctype(__urng()) - __urngmin;
while (__ret >= __past);
__ret /= __scaling;
}
}
else if (__urngrange < __urange)
{
__uctype __tmp;
do
{
const __uctype __uerngrange = __urngrange + 1;
__tmp = (__uerngrange * operator()
(__urng, param_type(0, __urange / __uerngrange)));
__ret = __tmp + (__uctype(__urng()) - __urngmin);
}
while (__ret > __urange || __ret < __tmp);
}
else
__ret = __uctype(__urng()) - __urngmin;
return __ret + __param.a();
}
template<typename _IntType>
template<typename _ForwardIterator,
typename _UniformRandomBitGenerator>
void
uniform_int_distribution<_IntType>::
__generate_impl(_ForwardIterator __f, _ForwardIterator __t,
_UniformRandomBitGenerator& __urng,
const param_type& __param)
{
typedef typename _UniformRandomBitGenerator::result_type _Gresult_type;
typedef typename make_unsigned<result_type>::type __utype;
typedef typename common_type<_Gresult_type, __utype>::type __uctype;
static_assert( __urng.min() < __urng.max(),
"Uniform random bit generator must define min() < max()");
constexpr __uctype __urngmin = __urng.min();
constexpr __uctype __urngmax = __urng.max();
constexpr __uctype __urngrange = __urngmax - __urngmin;
const __uctype __urange
= __uctype(__param.b()) - __uctype(__param.a());
__uctype __ret;
if (__urngrange > __urange)
{
if (__detail::_Power_of_2(__urngrange + 1)
&& __detail::_Power_of_2(__urange + 1))
{
while (__f != __t)
{
__ret = __uctype(__urng()) - __urngmin;
*__f++ = (__ret & __urange) + __param.a();
}
}
else
{
const __uctype __uerange = __urange + 1;
const __uctype __scaling = __urngrange / __uerange;
const __uctype __past = __uerange * __scaling;
while (__f != __t)
{
do
__ret = __uctype(__urng()) - __urngmin;
while (__ret >= __past);
*__f++ = __ret / __scaling + __param.a();
}
}
}
else if (__urngrange < __urange)
{
__uctype __tmp;
while (__f != __t)
{
do
{
constexpr __uctype __uerngrange = __urngrange + 1;
__tmp = (__uerngrange * operator()
(__urng, param_type(0, __urange / __uerngrange)));
__ret = __tmp + (__uctype(__urng()) - __urngmin);
}
while (__ret > __urange || __ret < __tmp);
*__f++ = __ret;
}
}
else
while (__f != __t)
*__f++ = __uctype(__urng()) - __urngmin + __param.a();
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<typename _Iterator, typename _Compare>
void
__move_median_to_first(_Iterator __result,_Iterator __a, _Iterator __b,
_Iterator __c, _Compare __comp)
{
if (__comp(__a, __b))
{
if (__comp(__b, __c))
std::iter_swap(__result, __b);
else if (__comp(__a, __c))
std::iter_swap(__result, __c);
else
std::iter_swap(__result, __a);
}
else if (__comp(__a, __c))
std::iter_swap(__result, __a);
else if (__comp(__b, __c))
std::iter_swap(__result, __c);
else
std::iter_swap(__result, __b);
}
template<typename _InputIterator, typename _Predicate>
inline _InputIterator
__find_if_not(_InputIterator __first, _InputIterator __last,
_Predicate __pred)
{
return std::__find_if(__first, __last,
__gnu_cxx::__ops::__negate(__pred),
std::__iterator_category(__first));
}
template<typename _InputIterator, typename _Predicate, typename _Distance>
_InputIterator
__find_if_not_n(_InputIterator __first, _Distance& __len, _Predicate __pred)
{
for (; __len; --__len, (void) ++__first)
if (!__pred(__first))
break;
return __first;
}
template<typename _ForwardIterator1, typename _ForwardIterator2,
typename _BinaryPredicate>
_ForwardIterator1
__search(_ForwardIterator1 __first1, _ForwardIterator1 __last1,
_ForwardIterator2 __first2, _ForwardIterator2 __last2,
_BinaryPredicate __predicate)
{
if (__first1 == __last1 || __first2 == __last2)
return __first1;
_ForwardIterator2 __p1(__first2);
if (++__p1 == __last2)
return std::__find_if(__first1, __last1,
__gnu_cxx::__ops::__iter_comp_iter(__predicate, __first2));
_ForwardIterator1 __current = __first1;
for (;;)
{
__first1 =
std::__find_if(__first1, __last1,
__gnu_cxx::__ops::__iter_comp_iter(__predicate, __first2));
if (__first1 == __last1)
return __last1;
_ForwardIterator2 __p = __p1;
__current = __first1;
if (++__current == __last1)
return __last1;
while (__predicate(__current, __p))
{
if (++__p == __last2)
return __first1;
if (++__current == __last1)
return __last1;
}
++__first1;
}
return __first1;
}
template<typename _ForwardIterator, typename _Integer,
typename _UnaryPredicate>
_ForwardIterator
__search_n_aux(_ForwardIterator __first, _ForwardIterator __last,
_Integer __count, _UnaryPredicate __unary_pred,
std::forward_iterator_tag)
{
__first = std::__find_if(__first, __last, __unary_pred);
while (__first != __last)
{
typename iterator_traits<_ForwardIterator>::difference_type
__n = __count;
_ForwardIterator __i = __first;
++__i;
while (__i != __last && __n != 1 && __unary_pred(__i))
{
++__i;
--__n;
}
if (__n == 1)
return __first;
if (__i == __last)
return __last;
__first = std::__find_if(++__i, __last, __unary_pred);
}
return __last;
}
template<typename _RandomAccessIter, typename _Integer,
typename _UnaryPredicate>
_RandomAccessIter
__search_n_aux(_RandomAccessIter __first, _RandomAccessIter __last,
_Integer __count, _UnaryPredicate __unary_pred,
std::random_access_iterator_tag)
{
typedef typename std::iterator_traits<_RandomAccessIter>::difference_type
_DistanceType;
_DistanceType __tailSize = __last - __first;
_DistanceType __remainder = __count;
while (__remainder <= __tailSize)
{
__first += __remainder;
__tailSize -= __remainder;
_RandomAccessIter __backTrack = __first;
while (__unary_pred(--__backTrack))
{
if (--__remainder == 0)
return (__first - __count);
}
__remainder = __count + 1 - (__first - __backTrack);
}
return __last;
}
template<typename _ForwardIterator, typename _Integer,
typename _UnaryPredicate>
_ForwardIterator
__search_n(_ForwardIterator __first, _ForwardIterator __last,
_Integer __count,
_UnaryPredicate __unary_pred)
{
if (__count <= 0)
return __first;
if (__count == 1)
return std::__find_if(__first, __last, __unary_pred);
return std::__search_n_aux(__first, __last, __count, __unary_pred,
std::__iterator_category(__first));
}
template<typename _ForwardIterator1, typename _ForwardIterator2,
typename _BinaryPredicate>
_ForwardIterator1
__find_end(_ForwardIterator1 __first1, _ForwardIterator1 __last1,
_ForwardIterator2 __first2, _ForwardIterator2 __last2,
forward_iterator_tag, forward_iterator_tag,
_BinaryPredicate __comp)
{
if (__first2 == __last2)
return __last1;
_ForwardIterator1 __result = __last1;
while (1)
{
_ForwardIterator1 __new_result
= std::__search(__first1, __last1, __first2, __last2, __comp);
if (__new_result == __last1)
return __result;
else
{
__result = __new_result;
__first1 = __new_result;
++__first1;
}
}
}
template<typename _BidirectionalIterator1, typename _BidirectionalIterator2,
typename _BinaryPredicate>
_BidirectionalIterator1
__find_end(_BidirectionalIterator1 __first1,
_BidirectionalIterator1 __last1,
_BidirectionalIterator2 __first2,
_BidirectionalIterator2 __last2,
bidirectional_iterator_tag, bidirectional_iterator_tag,
_BinaryPredicate __comp)
{
typedef reverse_iterator<_BidirectionalIterator1> _RevIterator1;
typedef reverse_iterator<_BidirectionalIterator2> _RevIterator2;
_RevIterator1 __rlast1(__first1);
_RevIterator2 __rlast2(__first2);
_RevIterator1 __rresult = std::__search(_RevIterator1(__last1), __rlast1,
_RevIterator2(__last2), __rlast2,
__comp);
if (__rresult == __rlast1)
return __last1;
else
{
_BidirectionalIterator1 __result = __rresult.base();
std::advance(__result, -std::distance(__first2, __last2));
return __result;
}
}
template<typename _ForwardIterator1, typename _ForwardIterator2>
inline _ForwardIterator1
find_end(_ForwardIterator1 __first1, _ForwardIterator1 __last1,
_ForwardIterator2 __first2, _ForwardIterator2 __last2)
{
;
;
return std::__find_end(__first1, __last1, __first2, __last2,
std::__iterator_category(__first1),
std::__iterator_category(__first2),
__gnu_cxx::__ops::__iter_equal_to_iter());
}
template<typename _ForwardIterator1, typename _ForwardIterator2,
typename _BinaryPredicate>
inline _ForwardIterator1
find_end(_ForwardIterator1 __first1, _ForwardIterator1 __last1,
_ForwardIterator2 __first2, _ForwardIterator2 __last2,
_BinaryPredicate __comp)
{
;
;
return std::__find_end(__first1, __last1, __first2, __last2,
std::__iterator_category(__first1),
std::__iterator_category(__first2),
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _InputIterator, typename _Predicate>
inline bool
all_of(_InputIterator __first, _InputIterator __last, _Predicate __pred)
{ return __last == std::find_if_not(__first, __last, __pred); }
template<typename _InputIterator, typename _Predicate>
inline bool
none_of(_InputIterator __first, _InputIterator __last, _Predicate __pred)
{ return __last == std::find_if(__first, __last, __pred); }
template<typename _InputIterator, typename _Predicate>
inline bool
any_of(_InputIterator __first, _InputIterator __last, _Predicate __pred)
{ return !std::none_of(__first, __last, __pred); }
template<typename _InputIterator, typename _Predicate>
inline _InputIterator
find_if_not(_InputIterator __first, _InputIterator __last,
_Predicate __pred)
{
;
return std::__find_if_not(__first, __last,
__gnu_cxx::__ops::__pred_iter(__pred));
}
template<typename _InputIterator, typename _Predicate>
inline bool
is_partitioned(_InputIterator __first, _InputIterator __last,
_Predicate __pred)
{
__first = std::find_if_not(__first, __last, __pred);
if (__first == __last)
return true;
++__first;
return std::none_of(__first, __last, __pred);
}
template<typename _ForwardIterator, typename _Predicate>
_ForwardIterator
partition_point(_ForwardIterator __first, _ForwardIterator __last,
_Predicate __pred)
{
;
typedef typename iterator_traits<_ForwardIterator>::difference_type
_DistanceType;
_DistanceType __len = std::distance(__first, __last);
while (__len > 0)
{
_DistanceType __half = __len >> 1;
_ForwardIterator __middle = __first;
std::advance(__middle, __half);
if (__pred(*__middle))
{
__first = __middle;
++__first;
__len = __len - __half - 1;
}
else
__len = __half;
}
return __first;
}
template<typename _InputIterator, typename _OutputIterator,
typename _Predicate>
_OutputIterator
__remove_copy_if(_InputIterator __first, _InputIterator __last,
_OutputIterator __result, _Predicate __pred)
{
for (; __first != __last; ++__first)
if (!__pred(__first))
{
*__result = *__first;
++__result;
}
return __result;
}
template<typename _InputIterator, typename _OutputIterator, typename _Tp>
inline _OutputIterator
remove_copy(_InputIterator __first, _InputIterator __last,
_OutputIterator __result, const _Tp& __value)
{
;
return std::__remove_copy_if(__first, __last, __result,
__gnu_cxx::__ops::__iter_equals_val(__value));
}
template<typename _InputIterator, typename _OutputIterator,
typename _Predicate>
inline _OutputIterator
remove_copy_if(_InputIterator __first, _InputIterator __last,
_OutputIterator __result, _Predicate __pred)
{
;
return std::__remove_copy_if(__first, __last, __result,
__gnu_cxx::__ops::__pred_iter(__pred));
}
template<typename _InputIterator, typename _OutputIterator,
typename _Predicate>
_OutputIterator
copy_if(_InputIterator __first, _InputIterator __last,
_OutputIterator __result, _Predicate __pred)
{
;
for (; __first != __last; ++__first)
if (__pred(*__first))
{
*__result = *__first;
++__result;
}
return __result;
}
template<typename _InputIterator, typename _Size, typename _OutputIterator>
_OutputIterator
__copy_n(_InputIterator __first, _Size __n,
_OutputIterator __result, input_iterator_tag)
{
return std::__niter_wrap(__result,
__copy_n_a(__first, __n,
std::__niter_base(__result), true));
}
template<typename _RandomAccessIterator, typename _Size,
typename _OutputIterator>
inline _OutputIterator
__copy_n(_RandomAccessIterator __first, _Size __n,
_OutputIterator __result, random_access_iterator_tag)
{ return std::copy(__first, __first + __n, __result); }
template<typename _InputIterator, typename _Size, typename _OutputIterator>
inline _OutputIterator
copy_n(_InputIterator __first, _Size __n, _OutputIterator __result)
{
const auto __n2 = std::__size_to_integer(__n);
if (__n2 <= 0)
return __result;
;
;
return std::__copy_n(__first, __n2, __result,
std::__iterator_category(__first));
}
template<typename _InputIterator, typename _OutputIterator1,
typename _OutputIterator2, typename _Predicate>
pair<_OutputIterator1, _OutputIterator2>
partition_copy(_InputIterator __first, _InputIterator __last,
_OutputIterator1 __out_true, _OutputIterator2 __out_false,
_Predicate __pred)
{
;
for (; __first != __last; ++__first)
if (__pred(*__first))
{
*__out_true = *__first;
++__out_true;
}
else
{
*__out_false = *__first;
++__out_false;
}
return pair<_OutputIterator1, _OutputIterator2>(__out_true, __out_false);
}
template<typename _ForwardIterator, typename _Tp>
inline _ForwardIterator
remove(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __value)
{
;
return std::__remove_if(__first, __last,
__gnu_cxx::__ops::__iter_equals_val(__value));
}
template<typename _ForwardIterator, typename _Predicate>
inline _ForwardIterator
remove_if(_ForwardIterator __first, _ForwardIterator __last,
_Predicate __pred)
{
;
return std::__remove_if(__first, __last,
__gnu_cxx::__ops::__pred_iter(__pred));
}
template<typename _ForwardIterator, typename _BinaryPredicate>
_ForwardIterator
__adjacent_find(_ForwardIterator __first, _ForwardIterator __last,
_BinaryPredicate __binary_pred)
{
if (__first == __last)
return __last;
_ForwardIterator __next = __first;
while (++__next != __last)
{
if (__binary_pred(__first, __next))
return __first;
__first = __next;
}
return __last;
}
template<typename _ForwardIterator, typename _BinaryPredicate>
_ForwardIterator
__unique(_ForwardIterator __first, _ForwardIterator __last,
_BinaryPredicate __binary_pred)
{
__first = std::__adjacent_find(__first, __last, __binary_pred);
if (__first == __last)
return __last;
_ForwardIterator __dest = __first;
++__first;
while (++__first != __last)
if (!__binary_pred(__dest, __first))
*++__dest = std::move(*__first);
return ++__dest;
}
template<typename _ForwardIterator>
inline _ForwardIterator
unique(_ForwardIterator __first, _ForwardIterator __last)
{
;
return std::__unique(__first, __last,
__gnu_cxx::__ops::__iter_equal_to_iter());
}
template<typename _ForwardIterator, typename _BinaryPredicate>
inline _ForwardIterator
unique(_ForwardIterator __first, _ForwardIterator __last,
_BinaryPredicate __binary_pred)
{
;
return std::__unique(__first, __last,
__gnu_cxx::__ops::__iter_comp_iter(__binary_pred));
}
template<typename _ForwardIterator, typename _OutputIterator,
typename _BinaryPredicate>
_OutputIterator
__unique_copy(_ForwardIterator __first, _ForwardIterator __last,
_OutputIterator __result, _BinaryPredicate __binary_pred,
forward_iterator_tag, output_iterator_tag)
{
_ForwardIterator __next = __first;
*__result = *__first;
while (++__next != __last)
if (!__binary_pred(__first, __next))
{
__first = __next;
*++__result = *__first;
}
return ++__result;
}
template<typename _InputIterator, typename _OutputIterator,
typename _BinaryPredicate>
_OutputIterator
__unique_copy(_InputIterator __first, _InputIterator __last,
_OutputIterator __result, _BinaryPredicate __binary_pred,
input_iterator_tag, output_iterator_tag)
{
typename iterator_traits<_InputIterator>::value_type __value = *__first;
__decltype(__gnu_cxx::__ops::__iter_comp_val(__binary_pred))
__rebound_pred
= __gnu_cxx::__ops::__iter_comp_val(__binary_pred);
*__result = __value;
while (++__first != __last)
if (!__rebound_pred(__first, __value))
{
__value = *__first;
*++__result = __value;
}
return ++__result;
}
template<typename _InputIterator, typename _ForwardIterator,
typename _BinaryPredicate>
_ForwardIterator
__unique_copy(_InputIterator __first, _InputIterator __last,
_ForwardIterator __result, _BinaryPredicate __binary_pred,
input_iterator_tag, forward_iterator_tag)
{
*__result = *__first;
while (++__first != __last)
if (!__binary_pred(__result, __first))
*++__result = *__first;
return ++__result;
}
template<typename _BidirectionalIterator>
void
__reverse(_BidirectionalIterator __first, _BidirectionalIterator __last,
bidirectional_iterator_tag)
{
while (true)
if (__first == __last || __first == --__last)
return;
else
{
std::iter_swap(__first, __last);
++__first;
}
}
template<typename _RandomAccessIterator>
void
__reverse(_RandomAccessIterator __first, _RandomAccessIterator __last,
random_access_iterator_tag)
{
if (__first == __last)
return;
--__last;
while (__first < __last)
{
std::iter_swap(__first, __last);
++__first;
--__last;
}
}
template<typename _BidirectionalIterator>
inline void
reverse(_BidirectionalIterator __first, _BidirectionalIterator __last)
{
;
std::__reverse(__first, __last, std::__iterator_category(__first));
}
template<typename _BidirectionalIterator, typename _OutputIterator>
_OutputIterator
reverse_copy(_BidirectionalIterator __first, _BidirectionalIterator __last,
_OutputIterator __result)
{
;
while (__first != __last)
{
--__last;
*__result = *__last;
++__result;
}
return __result;
}
template<typename _EuclideanRingElement>
_EuclideanRingElement
__gcd(_EuclideanRingElement __m, _EuclideanRingElement __n)
{
while (__n != 0)
{
_EuclideanRingElement __t = __m % __n;
__m = __n;
__n = __t;
}
return __m;
}
inline namespace _V2 {
template<typename _ForwardIterator>
_ForwardIterator
__rotate(_ForwardIterator __first,
_ForwardIterator __middle,
_ForwardIterator __last,
forward_iterator_tag)
{
if (__first == __middle)
return __last;
else if (__last == __middle)
return __first;
_ForwardIterator __first2 = __middle;
do
{
std::iter_swap(__first, __first2);
++__first;
++__first2;
if (__first == __middle)
__middle = __first2;
}
while (__first2 != __last);
_ForwardIterator __ret = __first;
__first2 = __middle;
while (__first2 != __last)
{
std::iter_swap(__first, __first2);
++__first;
++__first2;
if (__first == __middle)
__middle = __first2;
else if (__first2 == __last)
__first2 = __middle;
}
return __ret;
}
template<typename _BidirectionalIterator>
_BidirectionalIterator
__rotate(_BidirectionalIterator __first,
_BidirectionalIterator __middle,
_BidirectionalIterator __last,
bidirectional_iterator_tag)
{
if (__first == __middle)
return __last;
else if (__last == __middle)
return __first;
std::__reverse(__first, __middle, bidirectional_iterator_tag());
std::__reverse(__middle, __last, bidirectional_iterator_tag());
while (__first != __middle && __middle != __last)
{
std::iter_swap(__first, --__last);
++__first;
}
if (__first == __middle)
{
std::__reverse(__middle, __last, bidirectional_iterator_tag());
return __last;
}
else
{
std::__reverse(__first, __middle, bidirectional_iterator_tag());
return __first;
}
}
template<typename _RandomAccessIterator>
_RandomAccessIterator
__rotate(_RandomAccessIterator __first,
_RandomAccessIterator __middle,
_RandomAccessIterator __last,
random_access_iterator_tag)
{
if (__first == __middle)
return __last;
else if (__last == __middle)
return __first;
typedef typename iterator_traits<_RandomAccessIterator>::difference_type
_Distance;
typedef typename iterator_traits<_RandomAccessIterator>::value_type
_ValueType;
_Distance __n = __last - __first;
_Distance __k = __middle - __first;
if (__k == __n - __k)
{
std::swap_ranges(__first, __middle, __middle);
return __middle;
}
_RandomAccessIterator __p = __first;
_RandomAccessIterator __ret = __first + (__last - __middle);
for (;;)
{
if (__k < __n - __k)
{
if (__is_pod(_ValueType) && __k == 1)
{
_ValueType __t = std::move(*__p);
std::move(__p + 1, __p + __n, __p);
*(__p + __n - 1) = std::move(__t);
return __ret;
}
_RandomAccessIterator __q = __p + __k;
for (_Distance __i = 0; __i < __n - __k; ++ __i)
{
std::iter_swap(__p, __q);
++__p;
++__q;
}
__n %= __k;
if (__n == 0)
return __ret;
std::swap(__n, __k);
__k = __n - __k;
}
else
{
__k = __n - __k;
if (__is_pod(_ValueType) && __k == 1)
{
_ValueType __t = std::move(*(__p + __n - 1));
std::move_backward(__p, __p + __n - 1, __p + __n);
*__p = std::move(__t);
return __ret;
}
_RandomAccessIterator __q = __p + __n;
__p = __q - __k;
for (_Distance __i = 0; __i < __n - __k; ++ __i)
{
--__p;
--__q;
std::iter_swap(__p, __q);
}
__n %= __k;
if (__n == 0)
return __ret;
std::swap(__n, __k);
}
}
}
template<typename _ForwardIterator>
inline _ForwardIterator
rotate(_ForwardIterator __first, _ForwardIterator __middle,
_ForwardIterator __last)
{
;
;
return std::__rotate(__first, __middle, __last,
std::__iterator_category(__first));
}
}
template<typename _ForwardIterator, typename _OutputIterator>
inline _OutputIterator
rotate_copy(_ForwardIterator __first, _ForwardIterator __middle,
_ForwardIterator __last, _OutputIterator __result)
{
;
;
return std::copy(__first, __middle,
std::copy(__middle, __last, __result));
}
template<typename _ForwardIterator, typename _Predicate>
_ForwardIterator
__partition(_ForwardIterator __first, _ForwardIterator __last,
_Predicate __pred, forward_iterator_tag)
{
if (__first == __last)
return __first;
while (__pred(*__first))
if (++__first == __last)
return __first;
_ForwardIterator __next = __first;
while (++__next != __last)
if (__pred(*__next))
{
std::iter_swap(__first, __next);
++__first;
}
return __first;
}
template<typename _BidirectionalIterator, typename _Predicate>
_BidirectionalIterator
__partition(_BidirectionalIterator __first, _BidirectionalIterator __last,
_Predicate __pred, bidirectional_iterator_tag)
{
while (true)
{
while (true)
if (__first == __last)
return __first;
else if (__pred(*__first))
++__first;
else
break;
--__last;
while (true)
if (__first == __last)
return __first;
else if (!bool(__pred(*__last)))
--__last;
else
break;
std::iter_swap(__first, __last);
++__first;
}
}
template<typename _ForwardIterator, typename _Pointer, typename _Predicate,
typename _Distance>
_ForwardIterator
__stable_partition_adaptive(_ForwardIterator __first,
_ForwardIterator __last,
_Predicate __pred, _Distance __len,
_Pointer __buffer,
_Distance __buffer_size)
{
if (__len == 1)
return __first;
if (__len <= __buffer_size)
{
_ForwardIterator __result1 = __first;
_Pointer __result2 = __buffer;
*__result2 = std::move(*__first);
++__result2;
++__first;
for (; __first != __last; ++__first)
if (__pred(__first))
{
*__result1 = std::move(*__first);
++__result1;
}
else
{
*__result2 = std::move(*__first);
++__result2;
}
std::move(__buffer, __result2, __result1);
return __result1;
}
_ForwardIterator __middle = __first;
std::advance(__middle, __len / 2);
_ForwardIterator __left_split =
std::__stable_partition_adaptive(__first, __middle, __pred,
__len / 2, __buffer,
__buffer_size);
_Distance __right_len = __len - __len / 2;
_ForwardIterator __right_split =
std::__find_if_not_n(__middle, __right_len, __pred);
if (__right_len)
__right_split =
std::__stable_partition_adaptive(__right_split, __last, __pred,
__right_len,
__buffer, __buffer_size);
return std::rotate(__left_split, __middle, __right_split);
}
template<typename _ForwardIterator, typename _Predicate>
_ForwardIterator
__stable_partition(_ForwardIterator __first, _ForwardIterator __last,
_Predicate __pred)
{
__first = std::__find_if_not(__first, __last, __pred);
if (__first == __last)
return __first;
typedef typename iterator_traits<_ForwardIterator>::value_type
_ValueType;
typedef typename iterator_traits<_ForwardIterator>::difference_type
_DistanceType;
_Temporary_buffer<_ForwardIterator, _ValueType>
__buf(__first, std::distance(__first, __last));
return
std::__stable_partition_adaptive(__first, __last, __pred,
_DistanceType(__buf.requested_size()),
__buf.begin(),
_DistanceType(__buf.size()));
}
template<typename _ForwardIterator, typename _Predicate>
inline _ForwardIterator
stable_partition(_ForwardIterator __first, _ForwardIterator __last,
_Predicate __pred)
{
;
return std::__stable_partition(__first, __last,
__gnu_cxx::__ops::__pred_iter(__pred));
}
template<typename _RandomAccessIterator, typename _Compare>
void
__heap_select(_RandomAccessIterator __first,
_RandomAccessIterator __middle,
_RandomAccessIterator __last, _Compare __comp)
{
std::__make_heap(__first, __middle, __comp);
for (_RandomAccessIterator __i = __middle; __i < __last; ++__i)
if (__comp(__i, __first))
std::__pop_heap(__first, __middle, __i, __comp);
}
template<typename _InputIterator, typename _RandomAccessIterator,
typename _Compare>
_RandomAccessIterator
__partial_sort_copy(_InputIterator __first, _InputIterator __last,
_RandomAccessIterator __result_first,
_RandomAccessIterator __result_last,
_Compare __comp)
{
typedef typename iterator_traits<_InputIterator>::value_type
_InputValueType;
typedef iterator_traits<_RandomAccessIterator> _RItTraits;
typedef typename _RItTraits::difference_type _DistanceType;
if (__result_first == __result_last)
return __result_last;
_RandomAccessIterator __result_real_last = __result_first;
while (__first != __last && __result_real_last != __result_last)
{
*__result_real_last = *__first;
++__result_real_last;
++__first;
}
std::__make_heap(__result_first, __result_real_last, __comp);
while (__first != __last)
{
if (__comp(__first, __result_first))
std::__adjust_heap(__result_first, _DistanceType(0),
_DistanceType(__result_real_last
- __result_first),
_InputValueType(*__first), __comp);
++__first;
}
std::__sort_heap(__result_first, __result_real_last, __comp);
return __result_real_last;
}
template<typename _InputIterator, typename _RandomAccessIterator>
inline _RandomAccessIterator
partial_sort_copy(_InputIterator __first, _InputIterator __last,
_RandomAccessIterator __result_first,
_RandomAccessIterator __result_last)
{
;
;
;
return std::__partial_sort_copy(__first, __last,
__result_first, __result_last,
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _InputIterator, typename _RandomAccessIterator,
typename _Compare>
inline _RandomAccessIterator
partial_sort_copy(_InputIterator __first, _InputIterator __last,
_RandomAccessIterator __result_first,
_RandomAccessIterator __result_last,
_Compare __comp)
{
;
;
;
return std::__partial_sort_copy(__first, __last,
__result_first, __result_last,
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _RandomAccessIterator, typename _Compare>
void
__unguarded_linear_insert(_RandomAccessIterator __last,
_Compare __comp)
{
typename iterator_traits<_RandomAccessIterator>::value_type
__val = std::move(*__last);
_RandomAccessIterator __next = __last;
--__next;
while (__comp(__val, __next))
{
*__last = std::move(*__next);
__last = __next;
--__next;
}
*__last = std::move(__val);
}
template<typename _RandomAccessIterator, typename _Compare>
void
__insertion_sort(_RandomAccessIterator __first,
_RandomAccessIterator __last, _Compare __comp)
{
if (__first == __last) return;
for (_RandomAccessIterator __i = __first + 1; __i != __last; ++__i)
{
if (__comp(__i, __first))
{
typename iterator_traits<_RandomAccessIterator>::value_type
__val = std::move(*__i);
std::move_backward(__first, __i, __i + 1);
*__first = std::move(__val);
}
else
std::__unguarded_linear_insert(__i,
__gnu_cxx::__ops::__val_comp_iter(__comp));
}
}
template<typename _RandomAccessIterator, typename _Compare>
inline void
__unguarded_insertion_sort(_RandomAccessIterator __first,
_RandomAccessIterator __last, _Compare __comp)
{
for (_RandomAccessIterator __i = __first; __i != __last; ++__i)
std::__unguarded_linear_insert(__i,
__gnu_cxx::__ops::__val_comp_iter(__comp));
}
enum { _S_threshold = 16 };
template<typename _RandomAccessIterator, typename _Compare>
void
__final_insertion_sort(_RandomAccessIterator __first,
_RandomAccessIterator __last, _Compare __comp)
{
if (__last - __first > int(_S_threshold))
{
std::__insertion_sort(__first, __first + int(_S_threshold), __comp);
std::__unguarded_insertion_sort(__first + int(_S_threshold), __last,
__comp);
}
else
std::__insertion_sort(__first, __last, __comp);
}
template<typename _RandomAccessIterator, typename _Compare>
_RandomAccessIterator
__unguarded_partition(_RandomAccessIterator __first,
_RandomAccessIterator __last,
_RandomAccessIterator __pivot, _Compare __comp)
{
while (true)
{
while (__comp(__first, __pivot))
++__first;
--__last;
while (__comp(__pivot, __last))
--__last;
if (!(__first < __last))
return __first;
std::iter_swap(__first, __last);
++__first;
}
}
template<typename _RandomAccessIterator, typename _Compare>
inline _RandomAccessIterator
__unguarded_partition_pivot(_RandomAccessIterator __first,
_RandomAccessIterator __last, _Compare __comp)
{
_RandomAccessIterator __mid = __first + (__last - __first) / 2;
std::__move_median_to_first(__first, __first + 1, __mid, __last - 1,
__comp);
return std::__unguarded_partition(__first + 1, __last, __first, __comp);
}
template<typename _RandomAccessIterator, typename _Compare>
inline void
__partial_sort(_RandomAccessIterator __first,
_RandomAccessIterator __middle,
_RandomAccessIterator __last,
_Compare __comp)
{
std::__heap_select(__first, __middle, __last, __comp);
std::__sort_heap(__first, __middle, __comp);
}
template<typename _RandomAccessIterator, typename _Size, typename _Compare>
void
__introsort_loop(_RandomAccessIterator __first,
_RandomAccessIterator __last,
_Size __depth_limit, _Compare __comp)
{
while (__last - __first > int(_S_threshold))
{
if (__depth_limit == 0)
{
std::__partial_sort(__first, __last, __last, __comp);
return;
}
--__depth_limit;
_RandomAccessIterator __cut =
std::__unguarded_partition_pivot(__first, __last, __comp);
std::__introsort_loop(__cut, __last, __depth_limit, __comp);
__last = __cut;
}
}
template<typename _RandomAccessIterator, typename _Compare>
inline void
__sort(_RandomAccessIterator __first, _RandomAccessIterator __last,
_Compare __comp)
{
if (__first != __last)
{
std::__introsort_loop(__first, __last,
std::__lg(__last - __first) * 2,
__comp);
std::__final_insertion_sort(__first, __last, __comp);
}
}
template<typename _RandomAccessIterator, typename _Size, typename _Compare>
void
__introselect(_RandomAccessIterator __first, _RandomAccessIterator __nth,
_RandomAccessIterator __last, _Size __depth_limit,
_Compare __comp)
{
while (__last - __first > 3)
{
if (__depth_limit == 0)
{
std::__heap_select(__first, __nth + 1, __last, __comp);
std::iter_swap(__first, __nth);
return;
}
--__depth_limit;
_RandomAccessIterator __cut =
std::__unguarded_partition_pivot(__first, __last, __comp);
if (__cut <= __nth)
__first = __cut;
else
__last = __cut;
}
std::__insertion_sort(__first, __last, __comp);
}
template<typename _ForwardIterator, typename _Tp, typename _Compare>
inline _ForwardIterator
lower_bound(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __val, _Compare __comp)
{
;
return std::__lower_bound(__first, __last, __val,
__gnu_cxx::__ops::__iter_comp_val(__comp));
}
template<typename _ForwardIterator, typename _Tp, typename _Compare>
_ForwardIterator
__upper_bound(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __val, _Compare __comp)
{
typedef typename iterator_traits<_ForwardIterator>::difference_type
_DistanceType;
_DistanceType __len = std::distance(__first, __last);
while (__len > 0)
{
_DistanceType __half = __len >> 1;
_ForwardIterator __middle = __first;
std::advance(__middle, __half);
if (__comp(__val, __middle))
__len = __half;
else
{
__first = __middle;
++__first;
__len = __len - __half - 1;
}
}
return __first;
}
template<typename _ForwardIterator, typename _Tp>
inline _ForwardIterator
upper_bound(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __val)
{
;
return std::__upper_bound(__first, __last, __val,
__gnu_cxx::__ops::__val_less_iter());
}
template<typename _ForwardIterator, typename _Tp, typename _Compare>
inline _ForwardIterator
upper_bound(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __val, _Compare __comp)
{
;
return std::__upper_bound(__first, __last, __val,
__gnu_cxx::__ops::__val_comp_iter(__comp));
}
template<typename _ForwardIterator, typename _Tp,
typename _CompareItTp, typename _CompareTpIt>
pair<_ForwardIterator, _ForwardIterator>
__equal_range(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __val,
_CompareItTp __comp_it_val, _CompareTpIt __comp_val_it)
{
typedef typename iterator_traits<_ForwardIterator>::difference_type
_DistanceType;
_DistanceType __len = std::distance(__first, __last);
while (__len > 0)
{
_DistanceType __half = __len >> 1;
_ForwardIterator __middle = __first;
std::advance(__middle, __half);
if (__comp_it_val(__middle, __val))
{
__first = __middle;
++__first;
__len = __len - __half - 1;
}
else if (__comp_val_it(__val, __middle))
__len = __half;
else
{
_ForwardIterator __left
= std::__lower_bound(__first, __middle, __val, __comp_it_val);
std::advance(__first, __len);
_ForwardIterator __right
= std::__upper_bound(++__middle, __first, __val, __comp_val_it);
return pair<_ForwardIterator, _ForwardIterator>(__left, __right);
}
}
return pair<_ForwardIterator, _ForwardIterator>(__first, __first);
}
template<typename _ForwardIterator, typename _Tp>
inline pair<_ForwardIterator, _ForwardIterator>
equal_range(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __val)
{
;
;
return std::__equal_range(__first, __last, __val,
__gnu_cxx::__ops::__iter_less_val(),
__gnu_cxx::__ops::__val_less_iter());
}
template<typename _ForwardIterator, typename _Tp, typename _Compare>
inline pair<_ForwardIterator, _ForwardIterator>
equal_range(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __val, _Compare __comp)
{
;
;
return std::__equal_range(__first, __last, __val,
__gnu_cxx::__ops::__iter_comp_val(__comp),
__gnu_cxx::__ops::__val_comp_iter(__comp));
}
template<typename _ForwardIterator, typename _Tp>
bool
binary_search(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __val)
{
;
;
_ForwardIterator __i
= std::__lower_bound(__first, __last, __val,
__gnu_cxx::__ops::__iter_less_val());
return __i != __last && !(__val < *__i);
}
template<typename _ForwardIterator, typename _Tp, typename _Compare>
bool
binary_search(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __val, _Compare __comp)
{
;
;
_ForwardIterator __i
= std::__lower_bound(__first, __last, __val,
__gnu_cxx::__ops::__iter_comp_val(__comp));
return __i != __last && !bool(__comp(__val, *__i));
}
template<typename _InputIterator1, typename _InputIterator2,
typename _OutputIterator, typename _Compare>
void
__move_merge_adaptive(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_OutputIterator __result, _Compare __comp)
{
while (__first1 != __last1 && __first2 != __last2)
{
if (__comp(__first2, __first1))
{
*__result = std::move(*__first2);
++__first2;
}
else
{
*__result = std::move(*__first1);
++__first1;
}
++__result;
}
if (__first1 != __last1)
std::move(__first1, __last1, __result);
}
template<typename _BidirectionalIterator1, typename _BidirectionalIterator2,
typename _BidirectionalIterator3, typename _Compare>
void
__move_merge_adaptive_backward(_BidirectionalIterator1 __first1,
_BidirectionalIterator1 __last1,
_BidirectionalIterator2 __first2,
_BidirectionalIterator2 __last2,
_BidirectionalIterator3 __result,
_Compare __comp)
{
if (__first1 == __last1)
{
std::move_backward(__first2, __last2, __result);
return;
}
else if (__first2 == __last2)
return;
--__last1;
--__last2;
while (true)
{
if (__comp(__last2, __last1))
{
*--__result = std::move(*__last1);
if (__first1 == __last1)
{
std::move_backward(__first2, ++__last2, __result);
return;
}
--__last1;
}
else
{
*--__result = std::move(*__last2);
if (__first2 == __last2)
return;
--__last2;
}
}
}
template<typename _BidirectionalIterator1, typename _BidirectionalIterator2,
typename _Distance>
_BidirectionalIterator1
__rotate_adaptive(_BidirectionalIterator1 __first,
_BidirectionalIterator1 __middle,
_BidirectionalIterator1 __last,
_Distance __len1, _Distance __len2,
_BidirectionalIterator2 __buffer,
_Distance __buffer_size)
{
_BidirectionalIterator2 __buffer_end;
if (__len1 > __len2 && __len2 <= __buffer_size)
{
if (__len2)
{
__buffer_end = std::move(__middle, __last, __buffer);
std::move_backward(__first, __middle, __last);
return std::move(__buffer, __buffer_end, __first);
}
else
return __first;
}
else if (__len1 <= __buffer_size)
{
if (__len1)
{
__buffer_end = std::move(__first, __middle, __buffer);
std::move(__middle, __last, __first);
return std::move_backward(__buffer, __buffer_end, __last);
}
else
return __last;
}
else
return std::rotate(__first, __middle, __last);
}
template<typename _BidirectionalIterator, typename _Distance,
typename _Pointer, typename _Compare>
void
__merge_adaptive(_BidirectionalIterator __first,
_BidirectionalIterator __middle,
_BidirectionalIterator __last,
_Distance __len1, _Distance __len2,
_Pointer __buffer, _Compare __comp)
{
if (__len1 <= __len2)
{
_Pointer __buffer_end = std::move(__first, __middle, __buffer);
std::__move_merge_adaptive(__buffer, __buffer_end, __middle, __last,
__first, __comp);
}
else
{
_Pointer __buffer_end = std::move(__middle, __last, __buffer);
std::__move_merge_adaptive_backward(__first, __middle, __buffer,
__buffer_end, __last, __comp);
}
}
template<typename _BidirectionalIterator, typename _Distance,
typename _Pointer, typename _Compare>
void
__merge_adaptive_resize(_BidirectionalIterator __first,
_BidirectionalIterator __middle,
_BidirectionalIterator __last,
_Distance __len1, _Distance __len2,
_Pointer __buffer, _Distance __buffer_size,
_Compare __comp)
{
if (__len1 <= __buffer_size || __len2 <= __buffer_size)
std::__merge_adaptive(__first, __middle, __last,
__len1, __len2, __buffer, __comp);
else
{
_BidirectionalIterator __first_cut = __first;
_BidirectionalIterator __second_cut = __middle;
_Distance __len11 = 0;
_Distance __len22 = 0;
if (__len1 > __len2)
{
__len11 = __len1 / 2;
std::advance(__first_cut, __len11);
__second_cut
= std::__lower_bound(__middle, __last, *__first_cut,
__gnu_cxx::__ops::__iter_comp_val(__comp));
__len22 = std::distance(__middle, __second_cut);
}
else
{
__len22 = __len2 / 2;
std::advance(__second_cut, __len22);
__first_cut
= std::__upper_bound(__first, __middle, *__second_cut,
__gnu_cxx::__ops::__val_comp_iter(__comp));
__len11 = std::distance(__first, __first_cut);
}
_BidirectionalIterator __new_middle
= std::__rotate_adaptive(__first_cut, __middle, __second_cut,
_Distance(__len1 - __len11), __len22,
__buffer, __buffer_size);
std::__merge_adaptive_resize(__first, __first_cut, __new_middle,
__len11, __len22,
__buffer, __buffer_size, __comp);
std::__merge_adaptive_resize(__new_middle, __second_cut, __last,
_Distance(__len1 - __len11),
_Distance(__len2 - __len22),
__buffer, __buffer_size, __comp);
}
}
template<typename _BidirectionalIterator, typename _Distance,
typename _Compare>
void
__merge_without_buffer(_BidirectionalIterator __first,
_BidirectionalIterator __middle,
_BidirectionalIterator __last,
_Distance __len1, _Distance __len2,
_Compare __comp)
{
if (__len1 == 0 || __len2 == 0)
return;
if (__len1 + __len2 == 2)
{
if (__comp(__middle, __first))
std::iter_swap(__first, __middle);
return;
}
_BidirectionalIterator __first_cut = __first;
_BidirectionalIterator __second_cut = __middle;
_Distance __len11 = 0;
_Distance __len22 = 0;
if (__len1 > __len2)
{
__len11 = __len1 / 2;
std::advance(__first_cut, __len11);
__second_cut
= std::__lower_bound(__middle, __last, *__first_cut,
__gnu_cxx::__ops::__iter_comp_val(__comp));
__len22 = std::distance(__middle, __second_cut);
}
else
{
__len22 = __len2 / 2;
std::advance(__second_cut, __len22);
__first_cut
= std::__upper_bound(__first, __middle, *__second_cut,
__gnu_cxx::__ops::__val_comp_iter(__comp));
__len11 = std::distance(__first, __first_cut);
}
_BidirectionalIterator __new_middle
= std::rotate(__first_cut, __middle, __second_cut);
std::__merge_without_buffer(__first, __first_cut, __new_middle,
__len11, __len22, __comp);
std::__merge_without_buffer(__new_middle, __second_cut, __last,
__len1 - __len11, __len2 - __len22, __comp);
}
template<typename _BidirectionalIterator, typename _Compare>
void
__inplace_merge(_BidirectionalIterator __first,
_BidirectionalIterator __middle,
_BidirectionalIterator __last,
_Compare __comp)
{
typedef typename iterator_traits<_BidirectionalIterator>::value_type
_ValueType;
typedef typename iterator_traits<_BidirectionalIterator>::difference_type
_DistanceType;
if (__first == __middle || __middle == __last)
return;
const _DistanceType __len1 = std::distance(__first, __middle);
const _DistanceType __len2 = std::distance(__middle, __last);
typedef _Temporary_buffer<_BidirectionalIterator, _ValueType> _TmpBuf;
_TmpBuf __buf(__first, std::min(__len1, __len2));
if (__builtin_expect(__buf.size() == __buf.requested_size(), true))
std::__merge_adaptive
(__first, __middle, __last, __len1, __len2, __buf.begin(), __comp);
else if (__builtin_expect(__buf.begin() == 0, false))
std::__merge_without_buffer
(__first, __middle, __last, __len1, __len2, __comp);
else
std::__merge_adaptive_resize
(__first, __middle, __last, __len1, __len2, __buf.begin(),
_DistanceType(__buf.size()), __comp);
}
template<typename _BidirectionalIterator>
inline void
inplace_merge(_BidirectionalIterator __first,
_BidirectionalIterator __middle,
_BidirectionalIterator __last)
{
;
;
;
std::__inplace_merge(__first, __middle, __last,
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _BidirectionalIterator, typename _Compare>
inline void
inplace_merge(_BidirectionalIterator __first,
_BidirectionalIterator __middle,
_BidirectionalIterator __last,
_Compare __comp)
{
;
;
;
std::__inplace_merge(__first, __middle, __last,
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _InputIterator, typename _OutputIterator,
typename _Compare>
_OutputIterator
__move_merge(_InputIterator __first1, _InputIterator __last1,
_InputIterator __first2, _InputIterator __last2,
_OutputIterator __result, _Compare __comp)
{
while (__first1 != __last1 && __first2 != __last2)
{
if (__comp(__first2, __first1))
{
*__result = std::move(*__first2);
++__first2;
}
else
{
*__result = std::move(*__first1);
++__first1;
}
++__result;
}
return std::move(__first2, __last2, std::move(__first1, __last1, __result))
;
}
template<typename _RandomAccessIterator1, typename _RandomAccessIterator2,
typename _Distance, typename _Compare>
void
__merge_sort_loop(_RandomAccessIterator1 __first,
_RandomAccessIterator1 __last,
_RandomAccessIterator2 __result, _Distance __step_size,
_Compare __comp)
{
const _Distance __two_step = 2 * __step_size;
while (__last - __first >= __two_step)
{
__result = std::__move_merge(__first, __first + __step_size,
__first + __step_size,
__first + __two_step,
__result, __comp);
__first += __two_step;
}
__step_size = std::min(_Distance(__last - __first), __step_size);
std::__move_merge(__first, __first + __step_size,
__first + __step_size, __last, __result, __comp);
}
template<typename _RandomAccessIterator, typename _Distance,
typename _Compare>
void
__chunk_insertion_sort(_RandomAccessIterator __first,
_RandomAccessIterator __last,
_Distance __chunk_size, _Compare __comp)
{
while (__last - __first >= __chunk_size)
{
std::__insertion_sort(__first, __first + __chunk_size, __comp);
__first += __chunk_size;
}
std::__insertion_sort(__first, __last, __comp);
}
enum { _S_chunk_size = 7 };
template<typename _RandomAccessIterator, typename _Pointer, typename _Compare>
void
__merge_sort_with_buffer(_RandomAccessIterator __first,
_RandomAccessIterator __last,
_Pointer __buffer, _Compare __comp)
{
typedef typename iterator_traits<_RandomAccessIterator>::difference_type
_Distance;
const _Distance __len = __last - __first;
const _Pointer __buffer_last = __buffer + __len;
_Distance __step_size = _S_chunk_size;
std::__chunk_insertion_sort(__first, __last, __step_size, __comp);
while (__step_size < __len)
{
std::__merge_sort_loop(__first, __last, __buffer,
__step_size, __comp);
__step_size *= 2;
std::__merge_sort_loop(__buffer, __buffer_last, __first,
__step_size, __comp);
__step_size *= 2;
}
}
template<typename _RandomAccessIterator, typename _Pointer, typename _Compare>
void
__stable_sort_adaptive(_RandomAccessIterator __first,
_RandomAccessIterator __middle,
_RandomAccessIterator __last,
_Pointer __buffer, _Compare __comp)
{
std::__merge_sort_with_buffer(__first, __middle, __buffer, __comp);
std::__merge_sort_with_buffer(__middle, __last, __buffer, __comp);
std::__merge_adaptive(__first, __middle, __last,
__middle - __first, __last - __middle,
__buffer, __comp);
}
template<typename _RandomAccessIterator, typename _Pointer,
typename _Distance, typename _Compare>
void
__stable_sort_adaptive_resize(_RandomAccessIterator __first,
_RandomAccessIterator __last,
_Pointer __buffer, _Distance __buffer_size,
_Compare __comp)
{
const _Distance __len = (__last - __first + 1) / 2;
const _RandomAccessIterator __middle = __first + __len;
if (__len > __buffer_size)
{
std::__stable_sort_adaptive_resize(__first, __middle, __buffer,
__buffer_size, __comp);
std::__stable_sort_adaptive_resize(__middle, __last, __buffer,
__buffer_size, __comp);
std::__merge_adaptive_resize(__first, __middle, __last,
_Distance(__middle - __first),
_Distance(__last - __middle),
__buffer, __buffer_size,
__comp);
}
else
std::__stable_sort_adaptive(__first, __middle, __last,
__buffer, __comp);
}
template<typename _RandomAccessIterator, typename _Compare>
void
__inplace_stable_sort(_RandomAccessIterator __first,
_RandomAccessIterator __last, _Compare __comp)
{
if (__last - __first < 15)
{
std::__insertion_sort(__first, __last, __comp);
return;
}
_RandomAccessIterator __middle = __first + (__last - __first) / 2;
std::__inplace_stable_sort(__first, __middle, __comp);
std::__inplace_stable_sort(__middle, __last, __comp);
std::__merge_without_buffer(__first, __middle, __last,
__middle - __first,
__last - __middle,
__comp);
}
template<typename _InputIterator1, typename _InputIterator2,
typename _Compare>
bool
__includes(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_Compare __comp)
{
while (__first1 != __last1 && __first2 != __last2)
{
if (__comp(__first2, __first1))
return false;
if (!__comp(__first1, __first2))
++__first2;
++__first1;
}
return __first2 == __last2;
}
template<typename _InputIterator1, typename _InputIterator2>
inline bool
includes(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2)
{
;
;
;
;
return std::__includes(__first1, __last1, __first2, __last2,
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _InputIterator1, typename _InputIterator2,
typename _Compare>
inline bool
includes(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_Compare __comp)
{
;
;
;
;
return std::__includes(__first1, __last1, __first2, __last2,
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _BidirectionalIterator, typename _Compare>
bool
__next_permutation(_BidirectionalIterator __first,
_BidirectionalIterator __last, _Compare __comp)
{
if (__first == __last)
return false;
_BidirectionalIterator __i = __first;
++__i;
if (__i == __last)
return false;
__i = __last;
--__i;
for(;;)
{
_BidirectionalIterator __ii = __i;
--__i;
if (__comp(__i, __ii))
{
_BidirectionalIterator __j = __last;
while (!__comp(__i, --__j))
{}
std::iter_swap(__i, __j);
std::__reverse(__ii, __last,
std::__iterator_category(__first));
return true;
}
if (__i == __first)
{
std::__reverse(__first, __last,
std::__iterator_category(__first));
return false;
}
}
}
template<typename _BidirectionalIterator>
inline bool
next_permutation(_BidirectionalIterator __first,
_BidirectionalIterator __last)
{
;
;
return std::__next_permutation
(__first, __last, __gnu_cxx::__ops::__iter_less_iter());
}
template<typename _BidirectionalIterator, typename _Compare>
inline bool
next_permutation(_BidirectionalIterator __first,
_BidirectionalIterator __last, _Compare __comp)
{
;
;
return std::__next_permutation
(__first, __last, __gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _BidirectionalIterator, typename _Compare>
bool
__prev_permutation(_BidirectionalIterator __first,
_BidirectionalIterator __last, _Compare __comp)
{
if (__first == __last)
return false;
_BidirectionalIterator __i = __first;
++__i;
if (__i == __last)
return false;
__i = __last;
--__i;
for(;;)
{
_BidirectionalIterator __ii = __i;
--__i;
if (__comp(__ii, __i))
{
_BidirectionalIterator __j = __last;
while (!__comp(--__j, __i))
{}
std::iter_swap(__i, __j);
std::__reverse(__ii, __last,
std::__iterator_category(__first));
return true;
}
if (__i == __first)
{
std::__reverse(__first, __last,
std::__iterator_category(__first));
return false;
}
}
}
template<typename _BidirectionalIterator>
inline bool
prev_permutation(_BidirectionalIterator __first,
_BidirectionalIterator __last)
{
;
;
return std::__prev_permutation(__first, __last,
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _BidirectionalIterator, typename _Compare>
inline bool
prev_permutation(_BidirectionalIterator __first,
_BidirectionalIterator __last, _Compare __comp)
{
;
;
return std::__prev_permutation(__first, __last,
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _InputIterator, typename _OutputIterator,
typename _Predicate, typename _Tp>
_OutputIterator
__replace_copy_if(_InputIterator __first, _InputIterator __last,
_OutputIterator __result,
_Predicate __pred, const _Tp& __new_value)
{
for (; __first != __last; ++__first, (void)++__result)
if (__pred(__first))
*__result = __new_value;
else
*__result = *__first;
return __result;
}
template<typename _InputIterator, typename _OutputIterator, typename _Tp>
inline _OutputIterator
replace_copy(_InputIterator __first, _InputIterator __last,
_OutputIterator __result,
const _Tp& __old_value, const _Tp& __new_value)
{
;
return std::__replace_copy_if(__first, __last, __result,
__gnu_cxx::__ops::__iter_equals_val(__old_value),
__new_value);
}
template<typename _InputIterator, typename _OutputIterator,
typename _Predicate, typename _Tp>
inline _OutputIterator
replace_copy_if(_InputIterator __first, _InputIterator __last,
_OutputIterator __result,
_Predicate __pred, const _Tp& __new_value)
{
;
return std::__replace_copy_if(__first, __last, __result,
__gnu_cxx::__ops::__pred_iter(__pred),
__new_value);
}
template<typename _ForwardIterator>
inline bool
is_sorted(_ForwardIterator __first, _ForwardIterator __last)
{ return std::is_sorted_until(__first, __last) == __last; }
template<typename _ForwardIterator, typename _Compare>
inline bool
is_sorted(_ForwardIterator __first, _ForwardIterator __last,
_Compare __comp)
{ return std::is_sorted_until(__first, __last, __comp) == __last; }
template<typename _ForwardIterator, typename _Compare>
_ForwardIterator
__is_sorted_until(_ForwardIterator __first, _ForwardIterator __last,
_Compare __comp)
{
if (__first == __last)
return __last;
_ForwardIterator __next = __first;
for (++__next; __next != __last; __first = __next, (void)++__next)
if (__comp(__next, __first))
return __next;
return __next;
}
template<typename _ForwardIterator>
inline _ForwardIterator
is_sorted_until(_ForwardIterator __first, _ForwardIterator __last)
{
;
;
return std::__is_sorted_until(__first, __last,
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _ForwardIterator, typename _Compare>
inline _ForwardIterator
is_sorted_until(_ForwardIterator __first, _ForwardIterator __last,
_Compare __comp)
{
;
;
return std::__is_sorted_until(__first, __last,
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _Tp>
constexpr
inline pair<const _Tp&, const _Tp&>
minmax(const _Tp& __a, const _Tp& __b)
{
return __b < __a ? pair<const _Tp&, const _Tp&>(__b, __a)
: pair<const _Tp&, const _Tp&>(__a, __b);
}
template<typename _Tp, typename _Compare>
constexpr
inline pair<const _Tp&, const _Tp&>
minmax(const _Tp& __a, const _Tp& __b, _Compare __comp)
{
return __comp(__b, __a) ? pair<const _Tp&, const _Tp&>(__b, __a)
: pair<const _Tp&, const _Tp&>(__a, __b);
}
template<typename _ForwardIterator, typename _Compare>
constexpr
pair<_ForwardIterator, _ForwardIterator>
__minmax_element(_ForwardIterator __first, _ForwardIterator __last,
_Compare __comp)
{
_ForwardIterator __next = __first;
if (__first == __last
|| ++__next == __last)
return std::make_pair(__first, __first);
_ForwardIterator __min{}, __max{};
if (__comp(__next, __first))
{
__min = __next;
__max = __first;
}
else
{
__min = __first;
__max = __next;
}
__first = __next;
++__first;
while (__first != __last)
{
__next = __first;
if (++__next == __last)
{
if (__comp(__first, __min))
__min = __first;
else if (!__comp(__first, __max))
__max = __first;
break;
}
if (__comp(__next, __first))
{
if (__comp(__next, __min))
__min = __next;
if (!__comp(__first, __max))
__max = __first;
}
else
{
if (__comp(__first, __min))
__min = __first;
if (!__comp(__next, __max))
__max = __next;
}
__first = __next;
++__first;
}
return std::make_pair(__min, __max);
}
template<typename _ForwardIterator>
constexpr
inline pair<_ForwardIterator, _ForwardIterator>
minmax_element(_ForwardIterator __first, _ForwardIterator __last)
{
;
;
return std::__minmax_element(__first, __last,
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _ForwardIterator, typename _Compare>
constexpr
inline pair<_ForwardIterator, _ForwardIterator>
minmax_element(_ForwardIterator __first, _ForwardIterator __last,
_Compare __comp)
{
;
;
return std::__minmax_element(__first, __last,
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _Tp>
constexpr
inline pair<_Tp, _Tp>
minmax(initializer_list<_Tp> __l)
{
;
pair<const _Tp*, const _Tp*> __p =
std::__minmax_element(__l.begin(), __l.end(),
__gnu_cxx::__ops::__iter_less_iter());
return std::make_pair(*__p.first, *__p.second);
}
template<typename _Tp, typename _Compare>
constexpr
inline pair<_Tp, _Tp>
minmax(initializer_list<_Tp> __l, _Compare __comp)
{
;
pair<const _Tp*, const _Tp*> __p =
std::__minmax_element(__l.begin(), __l.end(),
__gnu_cxx::__ops::__iter_comp_iter(__comp));
return std::make_pair(*__p.first, *__p.second);
}
template<typename _ForwardIterator1, typename _ForwardIterator2,
typename _BinaryPredicate>
inline bool
is_permutation(_ForwardIterator1 __first1, _ForwardIterator1 __last1,
_ForwardIterator2 __first2, _BinaryPredicate __pred)
{
;
return std::__is_permutation(__first1, __last1, __first2,
__gnu_cxx::__ops::__iter_comp_iter(__pred));
}
template<typename _ForwardIterator1, typename _ForwardIterator2,
typename _BinaryPredicate>
bool
__is_permutation(_ForwardIterator1 __first1, _ForwardIterator1 __last1,
_ForwardIterator2 __first2, _ForwardIterator2 __last2,
_BinaryPredicate __pred)
{
using _Cat1
= typename iterator_traits<_ForwardIterator1>::iterator_category;
using _Cat2
= typename iterator_traits<_ForwardIterator2>::iterator_category;
using _It1_is_RA = is_same<_Cat1, random_access_iterator_tag>;
using _It2_is_RA = is_same<_Cat2, random_access_iterator_tag>;
constexpr bool __ra_iters = _It1_is_RA() && _It2_is_RA();
if (__ra_iters)
{
auto __d1 = std::distance(__first1, __last1);
auto __d2 = std::distance(__first2, __last2);
if (__d1 != __d2)
return false;
}
for (; __first1 != __last1 && __first2 != __last2;
++__first1, (void)++__first2)
if (!__pred(__first1, __first2))
break;
if (__ra_iters)
{
if (__first1 == __last1)
return true;
}
else
{
auto __d1 = std::distance(__first1, __last1);
auto __d2 = std::distance(__first2, __last2);
if (__d1 == 0 && __d2 == 0)
return true;
if (__d1 != __d2)
return false;
}
for (_ForwardIterator1 __scan = __first1; __scan != __last1; ++__scan)
{
if (__scan != std::__find_if(__first1, __scan,
__gnu_cxx::__ops::__iter_comp_iter(__pred, __scan)))
continue;
auto __matches = std::__count_if(__first2, __last2,
__gnu_cxx::__ops::__iter_comp_iter(__pred, __scan));
if (0 == __matches
|| std::__count_if(__scan, __last1,
__gnu_cxx::__ops::__iter_comp_iter(__pred, __scan))
!= __matches)
return false;
}
return true;
}
template<typename _ForwardIterator1, typename _ForwardIterator2>
inline bool
is_permutation(_ForwardIterator1 __first1, _ForwardIterator1 __last1,
_ForwardIterator2 __first2, _ForwardIterator2 __last2)
{
;
;
return
std::__is_permutation(__first1, __last1, __first2, __last2,
__gnu_cxx::__ops::__iter_equal_to_iter());
}
template<typename _ForwardIterator1, typename _ForwardIterator2,
typename _BinaryPredicate>
inline bool
is_permutation(_ForwardIterator1 __first1, _ForwardIterator1 __last1,
_ForwardIterator2 __first2, _ForwardIterator2 __last2,
_BinaryPredicate __pred)
{
;
;
return std::__is_permutation(__first1, __last1, __first2, __last2,
__gnu_cxx::__ops::__iter_comp_iter(__pred));
}
template<typename _Tp>
constexpr const _Tp&
clamp(const _Tp& __val, const _Tp& __lo, const _Tp& __hi)
{
do { if (std::__is_constant_evaluated() && !bool(!(__hi < __lo))) __builtin_unreachable(); } while (false);
return std::min(std::max(__val, __lo), __hi);
}
template<typename _Tp, typename _Compare>
constexpr const _Tp&
clamp(const _Tp& __val, const _Tp& __lo, const _Tp& __hi, _Compare __comp)
{
do { if (std::__is_constant_evaluated() && !bool(!__comp(__hi, __lo))) __builtin_unreachable(); } while (false);
return std::min(std::max(__val, __lo, __comp), __hi, __comp);
}
template<typename _IntType, typename _UniformRandomBitGenerator>
pair<_IntType, _IntType>
__gen_two_uniform_ints(_IntType __b0, _IntType __b1,
_UniformRandomBitGenerator&& __g)
{
_IntType __x
= uniform_int_distribution<_IntType>{0, (__b0 * __b1) - 1}(__g);
return std::make_pair(__x / __b1, __x % __b1);
}
template<typename _RandomAccessIterator,
typename _UniformRandomNumberGenerator>
void
shuffle(_RandomAccessIterator __first, _RandomAccessIterator __last,
_UniformRandomNumberGenerator&& __g)
{
;
if (__first == __last)
return;
typedef typename iterator_traits<_RandomAccessIterator>::difference_type
_DistanceType;
typedef typename std::make_unsigned<_DistanceType>::type __ud_type;
typedef typename std::uniform_int_distribution<__ud_type> __distr_type;
typedef typename __distr_type::param_type __p_type;
typedef typename remove_reference<_UniformRandomNumberGenerator>::type
_Gen;
typedef typename common_type<typename _Gen::result_type, __ud_type>::type
__uc_type;
const __uc_type __urngrange = __g.max() - __g.min();
const __uc_type __urange = __uc_type(__last - __first);
if (__urngrange / __urange >= __urange)
{
_RandomAccessIterator __i = __first + 1;
if ((__urange % 2) == 0)
{
__distr_type __d{0, 1};
std::iter_swap(__i++, __first + __d(__g));
}
while (__i != __last)
{
const __uc_type __swap_range = __uc_type(__i - __first) + 1;
const pair<__uc_type, __uc_type> __pospos =
__gen_two_uniform_ints(__swap_range, __swap_range + 1, __g);
std::iter_swap(__i++, __first + __pospos.first);
std::iter_swap(__i++, __first + __pospos.second);
}
return;
}
__distr_type __d;
for (_RandomAccessIterator __i = __first + 1; __i != __last; ++__i)
std::iter_swap(__i, __first + __d(__g, __p_type(0, __i - __first)));
}
template<typename _InputIterator, typename _Function>
_Function
for_each(_InputIterator __first, _InputIterator __last, _Function __f)
{
;
for (; __first != __last; ++__first)
__f(*__first);
return __f;
}
template<typename _InputIterator, typename _Size, typename _Function>
_InputIterator
for_each_n(_InputIterator __first, _Size __n, _Function __f)
{
auto __n2 = std::__size_to_integer(__n);
using _Cat = typename iterator_traits<_InputIterator>::iterator_category;
if constexpr (is_base_of_v<random_access_iterator_tag, _Cat>)
{
if (__n2 <= 0)
return __first;
auto __last = __first + __n2;
std::for_each(__first, __last, std::move(__f));
return __last;
}
else
{
while (__n2-->0)
{
__f(*__first);
++__first;
}
return __first;
}
}
template<typename _InputIterator, typename _Tp>
inline _InputIterator
find(_InputIterator __first, _InputIterator __last,
const _Tp& __val)
{
;
return std::__find_if(__first, __last,
__gnu_cxx::__ops::__iter_equals_val(__val));
}
template<typename _InputIterator, typename _Predicate>
inline _InputIterator
find_if(_InputIterator __first, _InputIterator __last,
_Predicate __pred)
{
;
return std::__find_if(__first, __last,
__gnu_cxx::__ops::__pred_iter(__pred));
}
template<typename _InputIterator, typename _ForwardIterator>
_InputIterator
find_first_of(_InputIterator __first1, _InputIterator __last1,
_ForwardIterator __first2, _ForwardIterator __last2)
{
;
;
for (; __first1 != __last1; ++__first1)
for (_ForwardIterator __iter = __first2; __iter != __last2; ++__iter)
if (*__first1 == *__iter)
return __first1;
return __last1;
}
template<typename _InputIterator, typename _ForwardIterator,
typename _BinaryPredicate>
_InputIterator
find_first_of(_InputIterator __first1, _InputIterator __last1,
_ForwardIterator __first2, _ForwardIterator __last2,
_BinaryPredicate __comp)
{
;
;
for (; __first1 != __last1; ++__first1)
for (_ForwardIterator __iter = __first2; __iter != __last2; ++__iter)
if (__comp(*__first1, *__iter))
return __first1;
return __last1;
}
template<typename _ForwardIterator>
inline _ForwardIterator
adjacent_find(_ForwardIterator __first, _ForwardIterator __last)
{
;
return std::__adjacent_find(__first, __last,
__gnu_cxx::__ops::__iter_equal_to_iter());
}
template<typename _ForwardIterator, typename _BinaryPredicate>
inline _ForwardIterator
adjacent_find(_ForwardIterator __first, _ForwardIterator __last,
_BinaryPredicate __binary_pred)
{
;
return std::__adjacent_find(__first, __last,
__gnu_cxx::__ops::__iter_comp_iter(__binary_pred));
}
template<typename _InputIterator, typename _Tp>
inline typename iterator_traits<_InputIterator>::difference_type
count(_InputIterator __first, _InputIterator __last, const _Tp& __value)
{
;
return std::__count_if(__first, __last,
__gnu_cxx::__ops::__iter_equals_val(__value));
}
template<typename _InputIterator, typename _Predicate>
inline typename iterator_traits<_InputIterator>::difference_type
count_if(_InputIterator __first, _InputIterator __last, _Predicate __pred)
{
;
return std::__count_if(__first, __last,
__gnu_cxx::__ops::__pred_iter(__pred));
}
template<typename _ForwardIterator1, typename _ForwardIterator2>
inline _ForwardIterator1
search(_ForwardIterator1 __first1, _ForwardIterator1 __last1,
_ForwardIterator2 __first2, _ForwardIterator2 __last2)
{
;
;
return std::__search(__first1, __last1, __first2, __last2,
__gnu_cxx::__ops::__iter_equal_to_iter());
}
template<typename _ForwardIterator1, typename _ForwardIterator2,
typename _BinaryPredicate>
inline _ForwardIterator1
search(_ForwardIterator1 __first1, _ForwardIterator1 __last1,
_ForwardIterator2 __first2, _ForwardIterator2 __last2,
_BinaryPredicate __predicate)
{
;
;
return std::__search(__first1, __last1, __first2, __last2,
__gnu_cxx::__ops::__iter_comp_iter(__predicate));
}
template<typename _ForwardIterator, typename _Integer, typename _Tp>
inline _ForwardIterator
search_n(_ForwardIterator __first, _ForwardIterator __last,
_Integer __count, const _Tp& __val)
{
;
return std::__search_n(__first, __last, __count,
__gnu_cxx::__ops::__iter_equals_val(__val));
}
template<typename _ForwardIterator, typename _Integer, typename _Tp,
typename _BinaryPredicate>
inline _ForwardIterator
search_n(_ForwardIterator __first, _ForwardIterator __last,
_Integer __count, const _Tp& __val,
_BinaryPredicate __binary_pred)
{
;
return std::__search_n(__first, __last, __count,
__gnu_cxx::__ops::__iter_comp_val(__binary_pred, __val));
}
template<typename _ForwardIterator, typename _Searcher>
inline _ForwardIterator
search(_ForwardIterator __first, _ForwardIterator __last,
const _Searcher& __searcher)
{ return __searcher(__first, __last).first; }
template<typename _InputIterator, typename _OutputIterator,
typename _UnaryOperation>
_OutputIterator
transform(_InputIterator __first, _InputIterator __last,
_OutputIterator __result, _UnaryOperation __unary_op)
{
;
for (; __first != __last; ++__first, (void)++__result)
*__result = __unary_op(*__first);
return __result;
}
template<typename _InputIterator1, typename _InputIterator2,
typename _OutputIterator, typename _BinaryOperation>
_OutputIterator
transform(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _OutputIterator __result,
_BinaryOperation __binary_op)
{
;
for (; __first1 != __last1; ++__first1, (void)++__first2, ++__result)
*__result = __binary_op(*__first1, *__first2);
return __result;
}
template<typename _ForwardIterator, typename _Tp>
void
replace(_ForwardIterator __first, _ForwardIterator __last,
const _Tp& __old_value, const _Tp& __new_value)
{
;
for (; __first != __last; ++__first)
if (*__first == __old_value)
*__first = __new_value;
}
template<typename _ForwardIterator, typename _Predicate, typename _Tp>
void
replace_if(_ForwardIterator __first, _ForwardIterator __last,
_Predicate __pred, const _Tp& __new_value)
{
;
for (; __first != __last; ++__first)
if (__pred(*__first))
*__first = __new_value;
}
template<typename _ForwardIterator, typename _Generator>
void
generate(_ForwardIterator __first, _ForwardIterator __last,
_Generator __gen)
{
;
for (; __first != __last; ++__first)
*__first = __gen();
}
template<typename _OutputIterator, typename _Size, typename _Generator>
_OutputIterator
generate_n(_OutputIterator __first, _Size __n, _Generator __gen)
{
typedef __decltype(std::__size_to_integer(__n)) _IntSize;
for (_IntSize __niter = std::__size_to_integer(__n);
__niter > 0; --__niter, (void) ++__first)
*__first = __gen();
return __first;
}
template<typename _InputIterator, typename _OutputIterator>
inline _OutputIterator
unique_copy(_InputIterator __first, _InputIterator __last,
_OutputIterator __result)
{
;
if (__first == __last)
return __result;
return std::__unique_copy(__first, __last, __result,
__gnu_cxx::__ops::__iter_equal_to_iter(),
std::__iterator_category(__first),
std::__iterator_category(__result));
}
template<typename _InputIterator, typename _OutputIterator,
typename _BinaryPredicate>
inline _OutputIterator
unique_copy(_InputIterator __first, _InputIterator __last,
_OutputIterator __result,
_BinaryPredicate __binary_pred)
{
;
if (__first == __last)
return __result;
return std::__unique_copy(__first, __last, __result,
__gnu_cxx::__ops::__iter_comp_iter(__binary_pred),
std::__iterator_category(__first),
std::__iterator_category(__result));
}
template<typename _RandomAccessIterator>
__attribute__ ((__deprecated__ ("use '" "std::shuffle" "' instead")))
inline void
random_shuffle(_RandomAccessIterator __first, _RandomAccessIterator __last)
{
;
if (__first != __last)
for (_RandomAccessIterator __i = __first + 1; __i != __last; ++__i)
{
_RandomAccessIterator __j = __first
+ std::rand() % ((__i - __first) + 1);
if (__i != __j)
std::iter_swap(__i, __j);
}
}
template<typename _RandomAccessIterator, typename _RandomNumberGenerator>
__attribute__ ((__deprecated__ ("use '" "std::shuffle" "' instead")))
void
random_shuffle(_RandomAccessIterator __first, _RandomAccessIterator __last,
_RandomNumberGenerator&& __rand)
{
;
if (__first == __last)
return;
for (_RandomAccessIterator __i = __first + 1; __i != __last; ++__i)
{
_RandomAccessIterator __j = __first + __rand((__i - __first) + 1);
if (__i != __j)
std::iter_swap(__i, __j);
}
}
template<typename _ForwardIterator, typename _Predicate>
inline _ForwardIterator
partition(_ForwardIterator __first, _ForwardIterator __last,
_Predicate __pred)
{
;
return std::__partition(__first, __last, __pred,
std::__iterator_category(__first));
}
template<typename _RandomAccessIterator>
inline void
partial_sort(_RandomAccessIterator __first,
_RandomAccessIterator __middle,
_RandomAccessIterator __last)
{
;
;
;
std::__partial_sort(__first, __middle, __last,
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _RandomAccessIterator, typename _Compare>
inline void
partial_sort(_RandomAccessIterator __first,
_RandomAccessIterator __middle,
_RandomAccessIterator __last,
_Compare __comp)
{
;
;
;
std::__partial_sort(__first, __middle, __last,
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _RandomAccessIterator>
inline void
nth_element(_RandomAccessIterator __first, _RandomAccessIterator __nth,
_RandomAccessIterator __last)
{
;
;
;
if (__first == __last || __nth == __last)
return;
std::__introselect(__first, __nth, __last,
std::__lg(__last - __first) * 2,
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _RandomAccessIterator, typename _Compare>
inline void
nth_element(_RandomAccessIterator __first, _RandomAccessIterator __nth,
_RandomAccessIterator __last, _Compare __comp)
{
;
;
;
if (__first == __last || __nth == __last)
return;
std::__introselect(__first, __nth, __last,
std::__lg(__last - __first) * 2,
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _RandomAccessIterator>
inline void
sort(_RandomAccessIterator __first, _RandomAccessIterator __last)
{
;
;
std::__sort(__first, __last, __gnu_cxx::__ops::__iter_less_iter());
}
template<typename _RandomAccessIterator, typename _Compare>
inline void
sort(_RandomAccessIterator __first, _RandomAccessIterator __last,
_Compare __comp)
{
;
;
std::__sort(__first, __last, __gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _InputIterator1, typename _InputIterator2,
typename _OutputIterator, typename _Compare>
_OutputIterator
__merge(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_OutputIterator __result, _Compare __comp)
{
while (__first1 != __last1 && __first2 != __last2)
{
if (__comp(__first2, __first1))
{
*__result = *__first2;
++__first2;
}
else
{
*__result = *__first1;
++__first1;
}
++__result;
}
return std::copy(__first2, __last2,
std::copy(__first1, __last1, __result));
}
template<typename _InputIterator1, typename _InputIterator2,
typename _OutputIterator>
inline _OutputIterator
merge(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_OutputIterator __result)
{
;
;
;
;
return std::__merge(__first1, __last1,
__first2, __last2, __result,
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _InputIterator1, typename _InputIterator2,
typename _OutputIterator, typename _Compare>
inline _OutputIterator
merge(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_OutputIterator __result, _Compare __comp)
{
;
;
;
;
return std::__merge(__first1, __last1,
__first2, __last2, __result,
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _RandomAccessIterator, typename _Compare>
inline void
__stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last,
_Compare __comp)
{
typedef typename iterator_traits<_RandomAccessIterator>::value_type
_ValueType;
typedef typename iterator_traits<_RandomAccessIterator>::difference_type
_DistanceType;
if (__first == __last)
return;
typedef _Temporary_buffer<_RandomAccessIterator, _ValueType> _TmpBuf;
_TmpBuf __buf(__first, (__last - __first + 1) / 2);
if (__builtin_expect(__buf.requested_size() == __buf.size(), true))
std::__stable_sort_adaptive(__first,
__first + _DistanceType(__buf.size()),
__last, __buf.begin(), __comp);
else if (__builtin_expect(__buf.begin() == 0, false))
std::__inplace_stable_sort(__first, __last, __comp);
else
std::__stable_sort_adaptive_resize(__first, __last, __buf.begin(),
_DistanceType(__buf.size()), __comp);
}
template<typename _RandomAccessIterator>
inline void
stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last)
{
;
;
std::__stable_sort(__first, __last,
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _RandomAccessIterator, typename _Compare>
inline void
stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last,
_Compare __comp)
{
;
;
std::__stable_sort(__first, __last,
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _InputIterator1, typename _InputIterator2,
typename _OutputIterator,
typename _Compare>
_OutputIterator
__set_union(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_OutputIterator __result, _Compare __comp)
{
while (__first1 != __last1 && __first2 != __last2)
{
if (__comp(__first1, __first2))
{
*__result = *__first1;
++__first1;
}
else if (__comp(__first2, __first1))
{
*__result = *__first2;
++__first2;
}
else
{
*__result = *__first1;
++__first1;
++__first2;
}
++__result;
}
return std::copy(__first2, __last2,
std::copy(__first1, __last1, __result));
}
template<typename _InputIterator1, typename _InputIterator2,
typename _OutputIterator>
inline _OutputIterator
set_union(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_OutputIterator __result)
{
;
;
;
;
return std::__set_union(__first1, __last1,
__first2, __last2, __result,
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _InputIterator1, typename _InputIterator2,
typename _OutputIterator, typename _Compare>
inline _OutputIterator
set_union(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_OutputIterator __result, _Compare __comp)
{
;
;
;
;
return std::__set_union(__first1, __last1,
__first2, __last2, __result,
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _InputIterator1, typename _InputIterator2,
typename _OutputIterator,
typename _Compare>
_OutputIterator
__set_intersection(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_OutputIterator __result, _Compare __comp)
{
while (__first1 != __last1 && __first2 != __last2)
if (__comp(__first1, __first2))
++__first1;
else if (__comp(__first2, __first1))
++__first2;
else
{
*__result = *__first1;
++__first1;
++__first2;
++__result;
}
return __result;
}
template<typename _InputIterator1, typename _InputIterator2,
typename _OutputIterator>
inline _OutputIterator
set_intersection(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_OutputIterator __result)
{
;
;
;
;
return std::__set_intersection(__first1, __last1,
__first2, __last2, __result,
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _InputIterator1, typename _InputIterator2,
typename _OutputIterator, typename _Compare>
inline _OutputIterator
set_intersection(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_OutputIterator __result, _Compare __comp)
{
;
;
;
;
return std::__set_intersection(__first1, __last1,
__first2, __last2, __result,
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _InputIterator1, typename _InputIterator2,
typename _OutputIterator,
typename _Compare>
_OutputIterator
__set_difference(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_OutputIterator __result, _Compare __comp)
{
while (__first1 != __last1 && __first2 != __last2)
if (__comp(__first1, __first2))
{
*__result = *__first1;
++__first1;
++__result;
}
else if (__comp(__first2, __first1))
++__first2;
else
{
++__first1;
++__first2;
}
return std::copy(__first1, __last1, __result);
}
template<typename _InputIterator1, typename _InputIterator2,
typename _OutputIterator>
inline _OutputIterator
set_difference(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_OutputIterator __result)
{
;
;
;
;
return std::__set_difference(__first1, __last1,
__first2, __last2, __result,
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _InputIterator1, typename _InputIterator2,
typename _OutputIterator, typename _Compare>
inline _OutputIterator
set_difference(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_OutputIterator __result, _Compare __comp)
{
;
;
;
;
return std::__set_difference(__first1, __last1,
__first2, __last2, __result,
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _InputIterator1, typename _InputIterator2,
typename _OutputIterator,
typename _Compare>
_OutputIterator
__set_symmetric_difference(_InputIterator1 __first1,
_InputIterator1 __last1,
_InputIterator2 __first2,
_InputIterator2 __last2,
_OutputIterator __result,
_Compare __comp)
{
while (__first1 != __last1 && __first2 != __last2)
if (__comp(__first1, __first2))
{
*__result = *__first1;
++__first1;
++__result;
}
else if (__comp(__first2, __first1))
{
*__result = *__first2;
++__first2;
++__result;
}
else
{
++__first1;
++__first2;
}
return std::copy(__first2, __last2,
std::copy(__first1, __last1, __result));
}
template<typename _InputIterator1, typename _InputIterator2,
typename _OutputIterator>
inline _OutputIterator
set_symmetric_difference(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_OutputIterator __result)
{
;
;
;
;
return std::__set_symmetric_difference(__first1, __last1,
__first2, __last2, __result,
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _InputIterator1, typename _InputIterator2,
typename _OutputIterator, typename _Compare>
inline _OutputIterator
set_symmetric_difference(_InputIterator1 __first1, _InputIterator1 __last1,
_InputIterator2 __first2, _InputIterator2 __last2,
_OutputIterator __result,
_Compare __comp)
{
;
;
;
;
return std::__set_symmetric_difference(__first1, __last1,
__first2, __last2, __result,
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _ForwardIterator, typename _Compare>
constexpr
_ForwardIterator
__min_element(_ForwardIterator __first, _ForwardIterator __last,
_Compare __comp)
{
if (__first == __last)
return __first;
_ForwardIterator __result = __first;
while (++__first != __last)
if (__comp(__first, __result))
__result = __first;
return __result;
}
template<typename _ForwardIterator>
constexpr
_ForwardIterator
inline min_element(_ForwardIterator __first, _ForwardIterator __last)
{
;
;
return std::__min_element(__first, __last,
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _ForwardIterator, typename _Compare>
constexpr
inline _ForwardIterator
min_element(_ForwardIterator __first, _ForwardIterator __last,
_Compare __comp)
{
;
;
return std::__min_element(__first, __last,
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _ForwardIterator, typename _Compare>
constexpr
_ForwardIterator
__max_element(_ForwardIterator __first, _ForwardIterator __last,
_Compare __comp)
{
if (__first == __last) return __first;
_ForwardIterator __result = __first;
while (++__first != __last)
if (__comp(__result, __first))
__result = __first;
return __result;
}
template<typename _ForwardIterator>
constexpr
inline _ForwardIterator
max_element(_ForwardIterator __first, _ForwardIterator __last)
{
;
;
return std::__max_element(__first, __last,
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _ForwardIterator, typename _Compare>
constexpr
inline _ForwardIterator
max_element(_ForwardIterator __first, _ForwardIterator __last,
_Compare __comp)
{
;
;
return std::__max_element(__first, __last,
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _Tp>
constexpr
inline _Tp
min(initializer_list<_Tp> __l)
{
;
return *std::__min_element(__l.begin(), __l.end(),
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _Tp, typename _Compare>
constexpr
inline _Tp
min(initializer_list<_Tp> __l, _Compare __comp)
{
;
return *std::__min_element(__l.begin(), __l.end(),
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _Tp>
constexpr
inline _Tp
max(initializer_list<_Tp> __l)
{
;
return *std::__max_element(__l.begin(), __l.end(),
__gnu_cxx::__ops::__iter_less_iter());
}
template<typename _Tp, typename _Compare>
constexpr
inline _Tp
max(initializer_list<_Tp> __l, _Compare __comp)
{
;
return *std::__max_element(__l.begin(), __l.end(),
__gnu_cxx::__ops::__iter_comp_iter(__comp));
}
template<typename _InputIterator, typename _RandomAccessIterator,
typename _Size, typename _UniformRandomBitGenerator>
_RandomAccessIterator
__sample(_InputIterator __first, _InputIterator __last, input_iterator_tag,
_RandomAccessIterator __out, random_access_iterator_tag,
_Size __n, _UniformRandomBitGenerator&& __g)
{
using __distrib_type = uniform_int_distribution<_Size>;
using __param_type = typename __distrib_type::param_type;
__distrib_type __d{};
_Size __sample_sz = 0;
while (__first != __last && __sample_sz != __n)
{
__out[__sample_sz++] = *__first;
++__first;
}
for (auto __pop_sz = __sample_sz; __first != __last;
++__first, (void) ++__pop_sz)
{
const auto __k = __d(__g, __param_type{0, __pop_sz});
if (__k < __n)
__out[__k] = *__first;
}
return __out + __sample_sz;
}
template<typename _ForwardIterator, typename _OutputIterator, typename _Cat,
typename _Size, typename _UniformRandomBitGenerator>
_OutputIterator
__sample(_ForwardIterator __first, _ForwardIterator __last,
forward_iterator_tag,
_OutputIterator __out, _Cat,
_Size __n, _UniformRandomBitGenerator&& __g)
{
using __distrib_type = uniform_int_distribution<_Size>;
using __param_type = typename __distrib_type::param_type;
using _USize = make_unsigned_t<_Size>;
using _Gen = remove_reference_t<_UniformRandomBitGenerator>;
using __uc_type = common_type_t<typename _Gen::result_type, _USize>;
if (__first == __last)
return __out;
__distrib_type __d{};
_Size __unsampled_sz = std::distance(__first, __last);
__n = std::min(__n, __unsampled_sz);
const __uc_type __urngrange = __g.max() - __g.min();
if (__urngrange / __uc_type(__unsampled_sz) >= __uc_type(__unsampled_sz))
{
while (__n != 0 && __unsampled_sz >= 2)
{
const pair<_Size, _Size> __p =
__gen_two_uniform_ints(__unsampled_sz, __unsampled_sz - 1, __g);
--__unsampled_sz;
if (__p.first < __n)
{
*__out++ = *__first;
--__n;
}
++__first;
if (__n == 0) break;
--__unsampled_sz;
if (__p.second < __n)
{
*__out++ = *__first;
--__n;
}
++__first;
}
}
for (; __n != 0; ++__first)
if (__d(__g, __param_type{0, --__unsampled_sz}) < __n)
{
*__out++ = *__first;
--__n;
}
return __out;
}
template<typename _PopulationIterator, typename _SampleIterator,
typename _Distance, typename _UniformRandomBitGenerator>
_SampleIterator
sample(_PopulationIterator __first, _PopulationIterator __last,
_SampleIterator __out, _Distance __n,
_UniformRandomBitGenerator&& __g)
{
using __pop_cat = typename
std::iterator_traits<_PopulationIterator>::iterator_category;
using __samp_cat = typename
std::iterator_traits<_SampleIterator>::iterator_category;
static_assert(
__or_<is_convertible<__pop_cat, forward_iterator_tag>,
is_convertible<__samp_cat, random_access_iterator_tag>>::value,
"output range must use a RandomAccessIterator when input range"
" does not meet the ForwardIterator requirements");
static_assert(is_integral<_Distance>::value,
"sample size must be an integer type");
typename iterator_traits<_PopulationIterator>::difference_type __d = __n;
return std::
__sample(__first, __last, __pop_cat{}, __out, __samp_cat{}, __d,
std::forward<_UniformRandomBitGenerator>(__g));
}
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template<int _Num> struct _Placeholder { };
template<typename _Callable, typename... _Args>
inline invoke_result_t<_Callable, _Args...>
invoke(_Callable&& __fn, _Args&&... __args)
noexcept(is_nothrow_invocable_v<_Callable, _Args...>)
{
return std::__invoke(std::forward<_Callable>(__fn),
std::forward<_Args>(__args)...);
}
template<typename _MemFunPtr,
bool __is_mem_fn = is_member_function_pointer<_MemFunPtr>::value>
class _Mem_fn_base
: public _Mem_fn_traits<_MemFunPtr>::__maybe_type
{
using _Traits = _Mem_fn_traits<_MemFunPtr>;
using _Arity = typename _Traits::__arity;
using _Varargs = typename _Traits::__vararg;
template<typename _Func, typename... _BoundArgs>
friend struct _Bind_check_arity;
_MemFunPtr _M_pmf;
public:
using result_type = typename _Traits::__result_type;
explicit constexpr
_Mem_fn_base(_MemFunPtr __pmf) noexcept : _M_pmf(__pmf) { }
template<typename... _Args>
auto
operator()(_Args&&... __args) const
noexcept(noexcept(
std::__invoke(_M_pmf, std::forward<_Args>(__args)...)))
-> decltype(std::__invoke(_M_pmf, std::forward<_Args>(__args)...))
{ return std::__invoke(_M_pmf, std::forward<_Args>(__args)...); }
};
template<typename _MemObjPtr>
class _Mem_fn_base<_MemObjPtr, false>
{
using _Arity = integral_constant<size_t, 0>;
using _Varargs = false_type;
template<typename _Func, typename... _BoundArgs>
friend struct _Bind_check_arity;
_MemObjPtr _M_pm;
public:
explicit constexpr
_Mem_fn_base(_MemObjPtr __pm) noexcept : _M_pm(__pm) { }
template<typename _Tp>
auto
operator()(_Tp&& __obj) const
noexcept(noexcept(std::__invoke(_M_pm, std::forward<_Tp>(__obj))))
-> decltype(std::__invoke(_M_pm, std::forward<_Tp>(__obj)))
{ return std::__invoke(_M_pm, std::forward<_Tp>(__obj)); }
};
template<typename _MemberPointer>
struct _Mem_fn;
template<typename _Res, typename _Class>
struct _Mem_fn<_Res _Class::*>
: _Mem_fn_base<_Res _Class::*>
{
using _Mem_fn_base<_Res _Class::*>::_Mem_fn_base;
};
template<typename _Tp, typename _Class>
inline _Mem_fn<_Tp _Class::*>
mem_fn(_Tp _Class::* __pm) noexcept
{
return _Mem_fn<_Tp _Class::*>(__pm);
}
template<typename _Tp>
struct is_bind_expression
: public false_type { };
template<typename _Tp>
struct is_placeholder
: public integral_constant<int, 0>
{ };
template <typename _Tp> inline constexpr bool is_bind_expression_v
= is_bind_expression<_Tp>::value;
template <typename _Tp> inline constexpr int is_placeholder_v
= is_placeholder<_Tp>::value;
namespace placeholders
{
inline const _Placeholder<1> _1;
inline const _Placeholder<2> _2;
inline const _Placeholder<3> _3;
inline const _Placeholder<4> _4;
inline const _Placeholder<5> _5;
inline const _Placeholder<6> _6;
inline const _Placeholder<7> _7;
inline const _Placeholder<8> _8;
inline const _Placeholder<9> _9;
inline const _Placeholder<10> _10;
inline const _Placeholder<11> _11;
inline const _Placeholder<12> _12;
inline const _Placeholder<13> _13;
inline const _Placeholder<14> _14;
inline const _Placeholder<15> _15;
inline const _Placeholder<16> _16;
inline const _Placeholder<17> _17;
inline const _Placeholder<18> _18;
inline const _Placeholder<19> _19;
inline const _Placeholder<20> _20;
inline const _Placeholder<21> _21;
inline const _Placeholder<22> _22;
inline const _Placeholder<23> _23;
inline const _Placeholder<24> _24;
inline const _Placeholder<25> _25;
inline const _Placeholder<26> _26;
inline const _Placeholder<27> _27;
inline const _Placeholder<28> _28;
inline const _Placeholder<29> _29;
}
template<int _Num>
struct is_placeholder<_Placeholder<_Num> >
: public integral_constant<int, _Num>
{ };
template<int _Num>
struct is_placeholder<const _Placeholder<_Num> >
: public integral_constant<int, _Num>
{ };
template<std::size_t __i, typename _Tuple>
using _Safe_tuple_element_t
= typename enable_if<(__i < tuple_size<_Tuple>::value),
tuple_element<__i, _Tuple>>::type::type;
template<typename _Arg,
bool _IsBindExp = is_bind_expression<_Arg>::value,
bool _IsPlaceholder = (is_placeholder<_Arg>::value > 0)>
class _Mu;
template<typename _Tp>
class _Mu<reference_wrapper<_Tp>, false, false>
{
public:
template<typename _CVRef, typename _Tuple>
_Tp&
operator()(_CVRef& __arg, _Tuple&) const volatile
{ return __arg.get(); }
};
template<typename _Arg>
class _Mu<_Arg, true, false>
{
public:
template<typename _CVArg, typename... _Args>
auto
operator()(_CVArg& __arg,
tuple<_Args...>& __tuple) const volatile
-> decltype(__arg(declval<_Args>()...))
{
typedef typename _Build_index_tuple<sizeof...(_Args)>::__type
_Indexes;
return this->__call(__arg, __tuple, _Indexes());
}
private:
template<typename _CVArg, typename... _Args, std::size_t... _Indexes>
auto
__call(_CVArg& __arg, tuple<_Args...>& __tuple,
const _Index_tuple<_Indexes...>&) const volatile
-> decltype(__arg(declval<_Args>()...))
{
return __arg(std::get<_Indexes>(std::move(__tuple))...);
}
};
template<typename _Arg>
class _Mu<_Arg, false, true>
{
public:
template<typename _Tuple>
_Safe_tuple_element_t<(is_placeholder<_Arg>::value - 1), _Tuple>&&
operator()(const volatile _Arg&, _Tuple& __tuple) const volatile
{
return
::std::get<(is_placeholder<_Arg>::value - 1)>(std::move(__tuple));
}
};
template<typename _Arg>
class _Mu<_Arg, false, false>
{
public:
template<typename _CVArg, typename _Tuple>
_CVArg&&
operator()(_CVArg&& __arg, _Tuple&) const volatile
{ return std::forward<_CVArg>(__arg); }
};
template<std::size_t _Ind, typename... _Tp>
inline auto
__volget(volatile tuple<_Tp...>& __tuple)
-> __tuple_element_t<_Ind, tuple<_Tp...>> volatile&
{ return std::get<_Ind>(const_cast<tuple<_Tp...>&>(__tuple)); }
template<std::size_t _Ind, typename... _Tp>
inline auto
__volget(const volatile tuple<_Tp...>& __tuple)
-> __tuple_element_t<_Ind, tuple<_Tp...>> const volatile&
{ return std::get<_Ind>(const_cast<const tuple<_Tp...>&>(__tuple)); }
template<typename _Signature>
class _Bind;
template<typename _Functor, typename... _Bound_args>
class _Bind<_Functor(_Bound_args...)>
: public _Weak_result_type<_Functor>
{
typedef typename _Build_index_tuple<sizeof...(_Bound_args)>::__type
_Bound_indexes;
_Functor _M_f;
tuple<_Bound_args...> _M_bound_args;
template<typename _Result, typename... _Args, std::size_t... _Indexes>
_Result
__call(tuple<_Args...>&& __args, _Index_tuple<_Indexes...>)
{
return std::__invoke(_M_f,
_Mu<_Bound_args>()(std::get<_Indexes>(_M_bound_args), __args)...
);
}
template<typename _Result, typename... _Args, std::size_t... _Indexes>
_Result
__call_c(tuple<_Args...>&& __args, _Index_tuple<_Indexes...>) const
{
return std::__invoke(_M_f,
_Mu<_Bound_args>()(std::get<_Indexes>(_M_bound_args), __args)...
);
}
template<typename _Result, typename... _Args, std::size_t... _Indexes>
_Result
__call_v(tuple<_Args...>&& __args,
_Index_tuple<_Indexes...>) volatile
{
return std::__invoke(_M_f,
_Mu<_Bound_args>()(__volget<_Indexes>(_M_bound_args), __args)...
);
}
template<typename _Result, typename... _Args, std::size_t... _Indexes>
_Result
__call_c_v(tuple<_Args...>&& __args,
_Index_tuple<_Indexes...>) const volatile
{
return std::__invoke(_M_f,
_Mu<_Bound_args>()(__volget<_Indexes>(_M_bound_args), __args)...
);
}
template<typename _BoundArg, typename _CallArgs>
using _Mu_type = decltype(
_Mu<typename remove_cv<_BoundArg>::type>()(
std::declval<_BoundArg&>(), std::declval<_CallArgs&>()) );
template<typename _Fn, typename _CallArgs, typename... _BArgs>
using _Res_type_impl
= typename result_of< _Fn&(_Mu_type<_BArgs, _CallArgs>&&...) >::type;
template<typename _CallArgs>
using _Res_type = _Res_type_impl<_Functor, _CallArgs, _Bound_args...>;
template<typename _CallArgs>
using __dependent = typename
enable_if<bool(tuple_size<_CallArgs>::value+1), _Functor>::type;
template<typename _CallArgs, template<class> class __cv_quals>
using _Res_type_cv = _Res_type_impl<
typename __cv_quals<__dependent<_CallArgs>>::type,
_CallArgs,
typename __cv_quals<_Bound_args>::type...>;
public:
template<typename... _Args>
explicit
_Bind(const _Functor& __f, _Args&&... __args)
: _M_f(__f), _M_bound_args(std::forward<_Args>(__args)...)
{ }
template<typename... _Args>
explicit
_Bind(_Functor&& __f, _Args&&... __args)
: _M_f(std::move(__f)), _M_bound_args(std::forward<_Args>(__args)...)
{ }
_Bind(const _Bind&) = default;
_Bind(_Bind&&) = default;
template<typename... _Args,
typename _Result = _Res_type<tuple<_Args...>>>
_Result
operator()(_Args&&... __args)
{
return this->__call<_Result>(
std::forward_as_tuple(std::forward<_Args>(__args)...),
_Bound_indexes());
}
template<typename... _Args,
typename _Result = _Res_type_cv<tuple<_Args...>, add_const>>
_Result
operator()(_Args&&... __args) const
{
return this->__call_c<_Result>(
std::forward_as_tuple(std::forward<_Args>(__args)...),
_Bound_indexes());
}
template<typename... _Args,
typename _Result = _Res_type_cv<tuple<_Args...>, add_volatile>>
[[deprecated("std::bind does not support volatile in C++17")]]
_Result
operator()(_Args&&... __args) volatile
{
return this->__call_v<_Result>(
std::forward_as_tuple(std::forward<_Args>(__args)...),
_Bound_indexes());
}
template<typename... _Args,
typename _Result = _Res_type_cv<tuple<_Args...>, add_cv>>
[[deprecated("std::bind does not support volatile in C++17")]]
_Result
operator()(_Args&&... __args) const volatile
{
return this->__call_c_v<_Result>(
std::forward_as_tuple(std::forward<_Args>(__args)...),
_Bound_indexes());
}
};
template<typename _Result, typename _Signature>
class _Bind_result;
template<typename _Result, typename _Functor, typename... _Bound_args>
class _Bind_result<_Result, _Functor(_Bound_args...)>
{
typedef typename _Build_index_tuple<sizeof...(_Bound_args)>::__type
_Bound_indexes;
_Functor _M_f;
tuple<_Bound_args...> _M_bound_args;
template<typename _Res, typename... _Args, std::size_t... _Indexes>
_Res
__call(tuple<_Args...>&& __args, _Index_tuple<_Indexes...>)
{
return std::__invoke_r<_Res>(_M_f, _Mu<_Bound_args>()
(std::get<_Indexes>(_M_bound_args), __args)...);
}
template<typename _Res, typename... _Args, std::size_t... _Indexes>
_Res
__call(tuple<_Args...>&& __args, _Index_tuple<_Indexes...>) const
{
return std::__invoke_r<_Res>(_M_f, _Mu<_Bound_args>()
(std::get<_Indexes>(_M_bound_args), __args)...);
}
template<typename _Res, typename... _Args, std::size_t... _Indexes>
_Res
__call(tuple<_Args...>&& __args, _Index_tuple<_Indexes...>) volatile
{
return std::__invoke_r<_Res>(_M_f, _Mu<_Bound_args>()
(__volget<_Indexes>(_M_bound_args), __args)...);
}
template<typename _Res, typename... _Args, std::size_t... _Indexes>
_Res
__call(tuple<_Args...>&& __args,
_Index_tuple<_Indexes...>) const volatile
{
return std::__invoke_r<_Res>(_M_f, _Mu<_Bound_args>()
(__volget<_Indexes>(_M_bound_args), __args)...);
}
public:
typedef _Result result_type;
template<typename... _Args>
explicit
_Bind_result(const _Functor& __f, _Args&&... __args)
: _M_f(__f), _M_bound_args(std::forward<_Args>(__args)...)
{ }
template<typename... _Args>
explicit
_Bind_result(_Functor&& __f, _Args&&... __args)
: _M_f(std::move(__f)), _M_bound_args(std::forward<_Args>(__args)...)
{ }
_Bind_result(const _Bind_result&) = default;
_Bind_result(_Bind_result&&) = default;
template<typename... _Args>
result_type
operator()(_Args&&... __args)
{
return this->__call<_Result>(
std::forward_as_tuple(std::forward<_Args>(__args)...),
_Bound_indexes());
}
template<typename... _Args>
result_type
operator()(_Args&&... __args) const
{
return this->__call<_Result>(
std::forward_as_tuple(std::forward<_Args>(__args)...),
_Bound_indexes());
}
template<typename... _Args>
[[deprecated("std::bind does not support volatile in C++17")]]
result_type
operator()(_Args&&... __args) volatile
{
return this->__call<_Result>(
std::forward_as_tuple(std::forward<_Args>(__args)...),
_Bound_indexes());
}
template<typename... _Args>
[[deprecated("std::bind does not support volatile in C++17")]]
result_type
operator()(_Args&&... __args) const volatile
{
return this->__call<_Result>(
std::forward_as_tuple(std::forward<_Args>(__args)...),
_Bound_indexes());
}
};
template<typename _Signature>
struct is_bind_expression<_Bind<_Signature> >
: public true_type { };
template<typename _Signature>
struct is_bind_expression<const _Bind<_Signature> >
: public true_type { };
template<typename _Signature>
struct is_bind_expression<volatile _Bind<_Signature> >
: public true_type { };
template<typename _Signature>
struct is_bind_expression<const volatile _Bind<_Signature>>
: public true_type { };
template<typename _Result, typename _Signature>
struct is_bind_expression<_Bind_result<_Result, _Signature>>
: public true_type { };
template<typename _Result, typename _Signature>
struct is_bind_expression<const _Bind_result<_Result, _Signature>>
: public true_type { };
template<typename _Result, typename _Signature>
struct is_bind_expression<volatile _Bind_result<_Result, _Signature>>
: public true_type { };
template<typename _Result, typename _Signature>
struct is_bind_expression<const volatile _Bind_result<_Result, _Signature>>
: public true_type { };
template<typename _Func, typename... _BoundArgs>
struct _Bind_check_arity { };
template<typename _Ret, typename... _Args, typename... _BoundArgs>
struct _Bind_check_arity<_Ret (*)(_Args...), _BoundArgs...>
{
static_assert(sizeof...(_BoundArgs) == sizeof...(_Args),
"Wrong number of arguments for function");
};
template<typename _Ret, typename... _Args, typename... _BoundArgs>
struct _Bind_check_arity<_Ret (*)(_Args......), _BoundArgs...>
{
static_assert(sizeof...(_BoundArgs) >= sizeof...(_Args),
"Wrong number of arguments for function");
};
template<typename _Tp, typename _Class, typename... _BoundArgs>
struct _Bind_check_arity<_Tp _Class::*, _BoundArgs...>
{
using _Arity = typename _Mem_fn<_Tp _Class::*>::_Arity;
using _Varargs = typename _Mem_fn<_Tp _Class::*>::_Varargs;
static_assert(_Varargs::value
? sizeof...(_BoundArgs) >= _Arity::value + 1
: sizeof...(_BoundArgs) == _Arity::value + 1,
"Wrong number of arguments for pointer-to-member");
};
template<typename _Tp, typename _Tp2 = typename decay<_Tp>::type>
using __is_socketlike = __or_<is_integral<_Tp2>, is_enum<_Tp2>>;
template<bool _SocketLike, typename _Func, typename... _BoundArgs>
struct _Bind_helper
: _Bind_check_arity<typename decay<_Func>::type, _BoundArgs...>
{
typedef typename decay<_Func>::type __func_type;
typedef _Bind<__func_type(typename decay<_BoundArgs>::type...)> type;
};
template<typename _Func, typename... _BoundArgs>
struct _Bind_helper<true, _Func, _BoundArgs...>
{ };
template<typename _Func, typename... _BoundArgs>
inline typename
_Bind_helper<__is_socketlike<_Func>::value, _Func, _BoundArgs...>::type
bind(_Func&& __f, _BoundArgs&&... __args)
{
typedef _Bind_helper<false, _Func, _BoundArgs...> __helper_type;
return typename __helper_type::type(std::forward<_Func>(__f),
std::forward<_BoundArgs>(__args)...);
}
template<typename _Result, typename _Func, typename... _BoundArgs>
struct _Bindres_helper
: _Bind_check_arity<typename decay<_Func>::type, _BoundArgs...>
{
typedef typename decay<_Func>::type __functor_type;
typedef _Bind_result<_Result,
__functor_type(typename decay<_BoundArgs>::type...)>
type;
};
template<typename _Result, typename _Func, typename... _BoundArgs>
inline
typename _Bindres_helper<_Result, _Func, _BoundArgs...>::type
bind(_Func&& __f, _BoundArgs&&... __args)
{
typedef _Bindres_helper<_Result, _Func, _BoundArgs...> __helper_type;
return typename __helper_type::type(std::forward<_Func>(__f),
std::forward<_BoundArgs>(__args)...);
}
template<typename _Fn>
class _Not_fn
{
template<typename _Fn2, typename... _Args>
using __inv_res_t = typename __invoke_result<_Fn2, _Args...>::type;
template<typename _Tp>
static decltype(!std::declval<_Tp>())
_S_not() noexcept(noexcept(!std::declval<_Tp>()));
public:
template<typename _Fn2>
constexpr
_Not_fn(_Fn2&& __fn, int)
: _M_fn(std::forward<_Fn2>(__fn)) { }
_Not_fn(const _Not_fn& __fn) = default;
_Not_fn(_Not_fn&& __fn) = default;
~_Not_fn() = default;
template<typename... _Args> decltype(_S_not<__inv_res_t<_Fn &, _Args...>>()) operator()(_Args&&... __args) & noexcept(__is_nothrow_invocable<_Fn &, _Args...>::value && noexcept(_S_not<__inv_res_t<_Fn &, _Args...>>())) { return !std::__invoke(std::forward< _Fn & >(_M_fn), std::forward<_Args>(__args)...); }
template<typename... _Args> decltype(_S_not<__inv_res_t<_Fn const &, _Args...>>()) operator()(_Args&&... __args) const & noexcept(__is_nothrow_invocable<_Fn const &, _Args...>::value && noexcept(_S_not<__inv_res_t<_Fn const &, _Args...>>())) { return !std::__invoke(std::forward< _Fn const & >(_M_fn), std::forward<_Args>(__args)...); }
template<typename... _Args> decltype(_S_not<__inv_res_t<_Fn &&, _Args...>>()) operator()(_Args&&... __args) && noexcept(__is_nothrow_invocable<_Fn &&, _Args...>::value && noexcept(_S_not<__inv_res_t<_Fn &&, _Args...>>())) { return !std::__invoke(std::forward< _Fn && >(_M_fn), std::forward<_Args>(__args)...); }
template<typename... _Args> decltype(_S_not<__inv_res_t<_Fn const &&, _Args...>>()) operator()(_Args&&... __args) const && noexcept(__is_nothrow_invocable<_Fn const &&, _Args...>::value && noexcept(_S_not<__inv_res_t<_Fn const &&, _Args...>>())) { return !std::__invoke(std::forward< _Fn const && >(_M_fn), std::forward<_Args>(__args)...); }
private:
_Fn _M_fn;
};
template<typename _Tp, typename _Pred>
struct __is_byte_like : false_type { };
template<typename _Tp>
struct __is_byte_like<_Tp, equal_to<_Tp>>
: __bool_constant<sizeof(_Tp) == 1 && is_integral<_Tp>::value> { };
template<typename _Tp>
struct __is_byte_like<_Tp, equal_to<void>>
: __bool_constant<sizeof(_Tp) == 1 && is_integral<_Tp>::value> { };
enum class byte : unsigned char;
template<>
struct __is_byte_like<byte, equal_to<byte>>
: true_type { };
template<>
struct __is_byte_like<byte, equal_to<void>>
: true_type { };
template<typename _Fn>
inline auto
not_fn(_Fn&& __fn)
noexcept(std::is_nothrow_constructible<std::decay_t<_Fn>, _Fn&&>::value)
{
return _Not_fn<std::decay_t<_Fn>>{std::forward<_Fn>(__fn), 0};
}
template<typename _ForwardIterator1, typename _BinaryPredicate = equal_to<>>
class default_searcher
{
public:
default_searcher(_ForwardIterator1 __pat_first,
_ForwardIterator1 __pat_last,
_BinaryPredicate __pred = _BinaryPredicate())
: _M_m(__pat_first, __pat_last, std::move(__pred))
{ }
template<typename _ForwardIterator2>
pair<_ForwardIterator2, _ForwardIterator2>
operator()(_ForwardIterator2 __first, _ForwardIterator2 __last) const
{
_ForwardIterator2 __first_ret =
std::search(__first, __last, std::get<0>(_M_m), std::get<1>(_M_m),
std::get<2>(_M_m));
auto __ret = std::make_pair(__first_ret, __first_ret);
if (__ret.first != __last)
std::advance(__ret.second, std::distance(std::get<0>(_M_m),
std::get<1>(_M_m)));
return __ret;
}
private:
tuple<_ForwardIterator1, _ForwardIterator1, _BinaryPredicate> _M_m;
};
template<typename _Key, typename _Tp, typename _Hash, typename _Pred>
struct __boyer_moore_map_base
{
template<typename _RAIter>
__boyer_moore_map_base(_RAIter __pat, size_t __patlen,
_Hash&& __hf, _Pred&& __pred)
: _M_bad_char{ __patlen, std::move(__hf), std::move(__pred) }
{
if (__patlen > 0)
for (__diff_type __i = 0; __i < __patlen - 1; ++__i)
_M_bad_char[__pat[__i]] = __patlen - 1 - __i;
}
using __diff_type = _Tp;
__diff_type
_M_lookup(_Key __key, __diff_type __not_found) const
{
auto __iter = _M_bad_char.find(__key);
if (__iter == _M_bad_char.end())
return __not_found;
return __iter->second;
}
_Pred
_M_pred() const { return _M_bad_char.key_eq(); }
std::unordered_map<_Key, _Tp, _Hash, _Pred> _M_bad_char;
};
template<typename _Tp, size_t _Len, typename _Pred>
struct __boyer_moore_array_base
{
template<typename _RAIter, typename _Unused>
__boyer_moore_array_base(_RAIter __pat, size_t __patlen,
_Unused&&, _Pred&& __pred)
: _M_bad_char{ array<_Tp, _Len>{}, std::move(__pred) }
{
std::get<0>(_M_bad_char).fill(__patlen);
if (__patlen > 0)
for (__diff_type __i = 0; __i < __patlen - 1; ++__i)
{
auto __ch = __pat[__i];
using _UCh = make_unsigned_t<decltype(__ch)>;
auto __uch = static_cast<_UCh>(__ch);
std::get<0>(_M_bad_char)[__uch] = __patlen - 1 - __i;
}
}
using __diff_type = _Tp;
template<typename _Key>
__diff_type
_M_lookup(_Key __key, __diff_type __not_found) const
{
auto __ukey = static_cast<make_unsigned_t<_Key>>(__key);
if (__ukey >= _Len)
return __not_found;
return std::get<0>(_M_bad_char)[__ukey];
}
const _Pred&
_M_pred() const { return std::get<1>(_M_bad_char); }
tuple<array<_Tp, _Len>, _Pred> _M_bad_char;
};
template<typename _RAIter, typename _Hash, typename _Pred,
typename _Val = typename iterator_traits<_RAIter>::value_type,
typename _Diff = typename iterator_traits<_RAIter>::difference_type>
using __boyer_moore_base_t
= __conditional_t<__is_byte_like<_Val, _Pred>::value,
__boyer_moore_array_base<_Diff, 256, _Pred>,
__boyer_moore_map_base<_Val, _Diff, _Hash, _Pred>>;
template<typename _RAIter, typename _Hash
= hash<typename iterator_traits<_RAIter>::value_type>,
typename _BinaryPredicate = equal_to<>>
class boyer_moore_searcher
: __boyer_moore_base_t<_RAIter, _Hash, _BinaryPredicate>
{
using _Base = __boyer_moore_base_t<_RAIter, _Hash, _BinaryPredicate>;
using typename _Base::__diff_type;
public:
boyer_moore_searcher(_RAIter __pat_first, _RAIter __pat_last,
_Hash __hf = _Hash(),
_BinaryPredicate __pred = _BinaryPredicate());
template<typename _RandomAccessIterator2>
pair<_RandomAccessIterator2, _RandomAccessIterator2>
operator()(_RandomAccessIterator2 __first,
_RandomAccessIterator2 __last) const;
private:
bool
_M_is_prefix(_RAIter __word, __diff_type __len,
__diff_type __pos)
{
const auto& __pred = this->_M_pred();
__diff_type __suffixlen = __len - __pos;
for (__diff_type __i = 0; __i < __suffixlen; ++__i)
if (!__pred(__word[__i], __word[__pos + __i]))
return false;
return true;
}
__diff_type
_M_suffix_length(_RAIter __word, __diff_type __len,
__diff_type __pos)
{
const auto& __pred = this->_M_pred();
__diff_type __i = 0;
while (__pred(__word[__pos - __i], __word[__len - 1 - __i])
&& __i < __pos)
{
++__i;
}
return __i;
}
template<typename _Tp>
__diff_type
_M_bad_char_shift(_Tp __c) const
{ return this->_M_lookup(__c, _M_pat_end - _M_pat); }
_RAIter _M_pat;
_RAIter _M_pat_end;
std::vector<__diff_type> _M_good_suffix;
};
template<typename _RAIter, typename _Hash
= hash<typename iterator_traits<_RAIter>::value_type>,
typename _BinaryPredicate = equal_to<>>
class boyer_moore_horspool_searcher
: __boyer_moore_base_t<_RAIter, _Hash, _BinaryPredicate>
{
using _Base = __boyer_moore_base_t<_RAIter, _Hash, _BinaryPredicate>;
using typename _Base::__diff_type;
public:
boyer_moore_horspool_searcher(_RAIter __pat,
_RAIter __pat_end,
_Hash __hf = _Hash(),
_BinaryPredicate __pred
= _BinaryPredicate())
: _Base(__pat, __pat_end - __pat, std::move(__hf), std::move(__pred)),
_M_pat(__pat), _M_pat_end(__pat_end)
{ }
template<typename _RandomAccessIterator2>
pair<_RandomAccessIterator2, _RandomAccessIterator2>
operator()(_RandomAccessIterator2 __first,
_RandomAccessIterator2 __last) const
{
const auto& __pred = this->_M_pred();
auto __patlen = _M_pat_end - _M_pat;
if (__patlen == 0)
return std::make_pair(__first, __first);
auto __len = __last - __first;
while (__len >= __patlen)
{
for (auto __scan = __patlen - 1;
__pred(__first[__scan], _M_pat[__scan]); --__scan)
if (__scan == 0)
return std::make_pair(__first, __first + __patlen);
auto __shift = _M_bad_char_shift(__first[__patlen - 1]);
__len -= __shift;
__first += __shift;
}
return std::make_pair(__last, __last);
}
private:
template<typename _Tp>
__diff_type
_M_bad_char_shift(_Tp __c) const
{ return this->_M_lookup(__c, _M_pat_end - _M_pat); }
_RAIter _M_pat;
_RAIter _M_pat_end;
};
template<typename _RAIter, typename _Hash, typename _BinaryPredicate>
boyer_moore_searcher<_RAIter, _Hash, _BinaryPredicate>::
boyer_moore_searcher(_RAIter __pat, _RAIter __pat_end,
_Hash __hf, _BinaryPredicate __pred)
: _Base(__pat, __pat_end - __pat, std::move(__hf), std::move(__pred)),
_M_pat(__pat), _M_pat_end(__pat_end), _M_good_suffix(__pat_end - __pat)
{
auto __patlen = __pat_end - __pat;
if (__patlen == 0)
return;
__diff_type __last_prefix = __patlen - 1;
for (__diff_type __p = __patlen - 1; __p >= 0; --__p)
{
if (_M_is_prefix(__pat, __patlen, __p + 1))
__last_prefix = __p + 1;
_M_good_suffix[__p] = __last_prefix + (__patlen - 1 - __p);
}
for (__diff_type __p = 0; __p < __patlen - 1; ++__p)
{
auto __slen = _M_suffix_length(__pat, __patlen, __p);
auto __pos = __patlen - 1 - __slen;
if (!__pred(__pat[__p - __slen], __pat[__pos]))
_M_good_suffix[__pos] = __patlen - 1 - __p + __slen;
}
}
template<typename _RAIter, typename _Hash, typename _BinaryPredicate>
template<typename _RandomAccessIterator2>
pair<_RandomAccessIterator2, _RandomAccessIterator2>
boyer_moore_searcher<_RAIter, _Hash, _BinaryPredicate>::
operator()(_RandomAccessIterator2 __first,
_RandomAccessIterator2 __last) const
{
auto __patlen = _M_pat_end - _M_pat;
if (__patlen == 0)
return std::make_pair(__first, __first);
const auto& __pred = this->_M_pred();
__diff_type __i = __patlen - 1;
auto __stringlen = __last - __first;
while (__i < __stringlen)
{
__diff_type __j = __patlen - 1;
while (__j >= 0 && __pred(__first[__i], _M_pat[__j]))
{
--__i;
--__j;
}
if (__j < 0)
{
const auto __match = __first + __i + 1;
return std::make_pair(__match, __match + __patlen);
}
__i += std::max(_M_bad_char_shift(__first[__i]),
_M_good_suffix[__j]);
}
return std::make_pair(__last, __last);
}
}
extern "C" {
extern "C" {
typedef __sig_atomic_t sig_atomic_t;
union sigval
{
int sival_int;
void *sival_ptr;
};
typedef union sigval __sigval_t;
typedef struct
{
int si_signo;
int si_errno;
int si_code;
union
{
int _pad[((128 / sizeof (int)) - 3)];
struct
{
__pid_t si_pid;
__uid_t si_uid;
} _kill;
struct
{
int si_tid;
int si_overrun;
__sigval_t si_sigval;
} _timer;
struct
{
__pid_t si_pid;
__uid_t si_uid;
__sigval_t si_sigval;
} _rt;
struct
{
__pid_t si_pid;
__uid_t si_uid;
int si_status;
__clock_t si_utime;
__clock_t si_stime;
} _sigchld;
struct
{
void *si_addr;
short int si_addr_lsb;
union
{
struct
{
void *_lower;
void *_upper;
} _addr_bnd;
__uint32_t _pkey;
} _bounds;
} _sigfault;
struct
{
long int si_band;
int si_fd;
} _sigpoll;
struct
{
void *_call_addr;
int _syscall;
unsigned int _arch;
} _sigsys;
} _sifields;
} siginfo_t ;
enum
{
SI_ASYNCNL = -60,
SI_DETHREAD = -7,
SI_TKILL,
SI_SIGIO,
SI_ASYNCIO,
SI_MESGQ,
SI_TIMER,
SI_QUEUE,
SI_USER,
SI_KERNEL = 0x80
};
enum
{
ILL_ILLOPC = 1,
ILL_ILLOPN,
ILL_ILLADR,
ILL_ILLTRP,
ILL_PRVOPC,
ILL_PRVREG,
ILL_COPROC,
ILL_BADSTK,
ILL_BADIADDR
};
enum
{
FPE_INTDIV = 1,
FPE_INTOVF,
FPE_FLTDIV,
FPE_FLTOVF,
FPE_FLTUND,
FPE_FLTRES,
FPE_FLTINV,
FPE_FLTSUB,
FPE_FLTUNK = 14,
FPE_CONDTRAP
};
enum
{
SEGV_MAPERR = 1,
SEGV_ACCERR,
SEGV_BNDERR,
SEGV_PKUERR,
SEGV_ACCADI,
SEGV_ADIDERR,
SEGV_ADIPERR,
SEGV_MTEAERR,
SEGV_MTESERR
};
enum
{
BUS_ADRALN = 1,
BUS_ADRERR,
BUS_OBJERR,
BUS_MCEERR_AR,
BUS_MCEERR_AO
};
enum
{
TRAP_BRKPT = 1,
TRAP_TRACE,
TRAP_BRANCH,
TRAP_HWBKPT,
TRAP_UNK
};
enum
{
CLD_EXITED = 1,
CLD_KILLED,
CLD_DUMPED,
CLD_TRAPPED,
CLD_STOPPED,
CLD_CONTINUED
};
enum
{
POLL_IN = 1,
POLL_OUT,
POLL_MSG,
POLL_ERR,
POLL_PRI,
POLL_HUP
};
typedef __sigval_t sigval_t;
typedef struct sigevent
{
__sigval_t sigev_value;
int sigev_signo;
int sigev_notify;
union
{
int _pad[((64 / sizeof (int)) - 3)];
__pid_t _tid;
struct
{
void (*_function) (__sigval_t);
pthread_attr_t *_attribute;
} _sigev_thread;
} _sigev_un;
} sigevent_t;
enum
{
SIGEV_SIGNAL = 0,
SIGEV_NONE,
SIGEV_THREAD,
SIGEV_THREAD_ID = 4
};
typedef void (*__sighandler_t) (int);
extern __sighandler_t __sysv_signal (int __sig, __sighandler_t __handler)
noexcept (true);
extern __sighandler_t sysv_signal (int __sig, __sighandler_t __handler)
noexcept (true);
extern __sighandler_t signal (int __sig, __sighandler_t __handler)
noexcept (true);
extern int kill (__pid_t __pid, int __sig) noexcept (true);
extern int killpg (__pid_t __pgrp, int __sig) noexcept (true);
extern int raise (int __sig) noexcept (true);
extern __sighandler_t ssignal (int __sig, __sighandler_t __handler)
noexcept (true);
extern int gsignal (int __sig) noexcept (true);
extern void psignal (int __sig, const char *__s);
extern void psiginfo (const siginfo_t *__pinfo, const char *__s);
extern int sigpause (int __sig) __asm__ ("__xpg_sigpause")
__attribute__ ((__deprecated__ ("Use the sigsuspend function instead")));
extern int sigblock (int __mask) noexcept (true) __attribute__ ((__deprecated__));
extern int sigsetmask (int __mask) noexcept (true) __attribute__ ((__deprecated__));
extern int siggetmask (void) noexcept (true) __attribute__ ((__deprecated__));
typedef __sighandler_t sighandler_t;
typedef __sighandler_t sig_t;
extern int sigemptyset (sigset_t *__set) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int sigfillset (sigset_t *__set) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int sigaddset (sigset_t *__set, int __signo) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int sigdelset (sigset_t *__set, int __signo) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int sigismember (const sigset_t *__set, int __signo)
noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int sigisemptyset (const sigset_t *__set) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int sigandset (sigset_t *__set, const sigset_t *__left,
const sigset_t *__right) noexcept (true) __attribute__ ((__nonnull__ (1, 2, 3)));
extern int sigorset (sigset_t *__set, const sigset_t *__left,
const sigset_t *__right) noexcept (true) __attribute__ ((__nonnull__ (1, 2, 3)));
struct sigaction
{
union
{
__sighandler_t sa_handler;
void (*sa_sigaction) (int, siginfo_t *, void *);
}
__sigaction_handler;
__sigset_t sa_mask;
int sa_flags;
void (*sa_restorer) (void);
};
extern int sigprocmask (int __how, const sigset_t *__restrict __set,
sigset_t *__restrict __oset) noexcept (true);
extern int sigsuspend (const sigset_t *__set) __attribute__ ((__nonnull__ (1)));
extern int sigaction (int __sig, const struct sigaction *__restrict __act,
struct sigaction *__restrict __oact) noexcept (true);
extern int sigpending (sigset_t *__set) noexcept (true) __attribute__ ((__nonnull__ (1)));
extern int sigwait (const sigset_t *__restrict __set, int *__restrict __sig)
__attribute__ ((__nonnull__ (1, 2)));
extern int sigwaitinfo (const sigset_t *__restrict __set,
siginfo_t *__restrict __info) __attribute__ ((__nonnull__ (1)));
extern int sigtimedwait (const sigset_t *__restrict __set,
siginfo_t *__restrict __info,
const struct timespec *__restrict __timeout)
__attribute__ ((__nonnull__ (1)));
extern int sigqueue (__pid_t __pid, int __sig, const union sigval __val)
noexcept (true);
struct _fpx_sw_bytes
{
__uint32_t magic1;
__uint32_t extended_size;
__uint64_t xstate_bv;
__uint32_t xstate_size;
__uint32_t __glibc_reserved1[7];
};
struct _fpreg
{
unsigned short significand[4];
unsigned short exponent;
};
struct _fpxreg
{
unsigned short significand[4];
unsigned short exponent;
unsigned short __glibc_reserved1[3];
};
struct _xmmreg
{
__uint32_t element[4];
};
struct _fpstate
{
__uint32_t cw;
__uint32_t sw;
__uint32_t tag;
__uint32_t ipoff;
__uint32_t cssel;
__uint32_t dataoff;
__uint32_t datasel;
struct _fpreg _st[8];
unsigned short status;
unsigned short magic;
__uint32_t _fxsr_env[6];
__uint32_t mxcsr;
__uint32_t __glibc_reserved1;
struct _fpxreg _fxsr_st[8];
struct _xmmreg _xmm[8];
__uint32_t __glibc_reserved2[56];
};
struct sigcontext
{
unsigned short gs, __gsh;
unsigned short fs, __fsh;
unsigned short es, __esh;
unsigned short ds, __dsh;
unsigned long edi;
unsigned long esi;
unsigned long ebp;
unsigned long esp;
unsigned long ebx;
unsigned long edx;
unsigned long ecx;
unsigned long eax;
unsigned long trapno;
unsigned long err;
unsigned long eip;
unsigned short cs, __csh;
unsigned long eflags;
unsigned long esp_at_signal;
unsigned short ss, __ssh;
struct _fpstate * fpstate;
unsigned long oldmask;
unsigned long cr2;
};
struct _xsave_hdr
{
__uint64_t xstate_bv;
__uint64_t __glibc_reserved1[2];
__uint64_t __glibc_reserved2[5];
};
struct _ymmh_state
{
__uint32_t ymmh_space[64];
};
struct _xstate
{
struct _fpstate fpstate;
struct _xsave_hdr xstate_hdr;
struct _ymmh_state ymmh;
};
extern int sigreturn (struct sigcontext *__scp) noexcept (true);
typedef struct
{
void *ss_sp;
int ss_flags;
size_t ss_size;
} stack_t;
typedef int greg_t;
typedef greg_t gregset_t[19];
enum
{
REG_GS = 0,
REG_FS,
REG_ES,
REG_DS,
REG_EDI,
REG_ESI,
REG_EBP,
REG_ESP,
REG_EBX,
REG_EDX,
REG_ECX,
REG_EAX,
REG_TRAPNO,
REG_ERR,
REG_EIP,
REG_CS,
REG_EFL,
REG_UESP,
REG_SS
};
struct _libc_fpreg
{
unsigned short int significand[4];
unsigned short int exponent;
};
struct _libc_fpstate
{
unsigned long int cw;
unsigned long int sw;
unsigned long int tag;
unsigned long int ipoff;
unsigned long int cssel;
unsigned long int dataoff;
unsigned long int datasel;
struct _libc_fpreg _st[8];
unsigned long int status;
};
typedef struct _libc_fpstate *fpregset_t;
typedef struct
{
gregset_t gregs;
fpregset_t fpregs;
unsigned long int oldmask;
unsigned long int cr2;
} mcontext_t;
typedef struct ucontext_t
{
unsigned long int uc_flags;
struct ucontext_t *uc_link;
stack_t uc_stack;
mcontext_t uc_mcontext;
sigset_t uc_sigmask;
struct _libc_fpstate __fpregs_mem;
unsigned long int __ssp[4];
} ucontext_t;
extern int siginterrupt (int __sig, int __interrupt) noexcept (true)
__attribute__ ((__deprecated__ ("Use sigaction with SA_RESTART instead")));
enum
{
SS_ONSTACK = 1,
SS_DISABLE
};
extern int sigaltstack (const stack_t *__restrict __ss,
stack_t *__restrict __oss) noexcept (true);
struct sigstack
{
void *ss_sp;
int ss_onstack;
};
extern int sigstack (struct sigstack *__ss, struct sigstack *__oss)
noexcept (true) __attribute__ ((__deprecated__));
extern int sighold (int __sig) noexcept (true)
__attribute__ ((__deprecated__ ("Use the sigprocmask function instead")));
extern int sigrelse (int __sig) noexcept (true)
__attribute__ ((__deprecated__ ("Use the sigprocmask function instead")));
extern int sigignore (int __sig) noexcept (true)
__attribute__ ((__deprecated__ ("Use the signal function instead")));
extern __sighandler_t sigset (int __sig, __sighandler_t __disp) noexcept (true)
__attribute__ ((__deprecated__ ("Use the signal and sigprocmask functions instead")))
;
extern int pthread_sigmask (int __how,
const __sigset_t *__restrict __newmask,
__sigset_t *__restrict __oldmask)noexcept (true);
extern int pthread_kill (pthread_t __threadid, int __signo) noexcept (true);
extern int pthread_sigqueue (pthread_t __threadid, int __signo,
const union sigval __value) noexcept (true);
extern int __libc_current_sigrtmin (void) noexcept (true);
extern int __libc_current_sigrtmax (void) noexcept (true);
extern int tgkill (__pid_t __tgid, __pid_t __tid, int __signal);
}
typedef enum
{
P_ALL,
P_PID,
P_PGID,
P_PIDFD,
} idtype_t;
extern __pid_t wait (int *__stat_loc);
extern __pid_t waitpid (__pid_t __pid, int *__stat_loc, int __options);
extern int waitid (idtype_t __idtype, __id_t __id, siginfo_t *__infop,
int __options);
struct rusage;
extern __pid_t wait3 (int *__stat_loc, int __options,
struct rusage * __usage) noexcept (true);
extern __pid_t wait4 (__pid_t __pid, int *__stat_loc, int __options,
struct rusage *__usage) noexcept (true);
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template <typename _Key, typename _Tp, typename _Compare, typename _Alloc>
class multimap;
template <typename _Key, typename _Tp, typename _Compare = std::less<_Key>,
typename _Alloc = std::allocator<std::pair<const _Key, _Tp> > >
class map
{
public:
typedef _Key key_type;
typedef _Tp mapped_type;
typedef std::pair<const _Key, _Tp> value_type;
typedef _Compare key_compare;
typedef _Alloc allocator_type;
private:
public:
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
class value_compare
: public std::binary_function<value_type, value_type, bool>
{
friend class map<_Key, _Tp, _Compare, _Alloc>;
protected:
_Compare comp;
value_compare(_Compare __c)
: comp(__c) { }
public:
bool operator()(const value_type& __x, const value_type& __y) const
{ return comp(__x.first, __y.first); }
};
#pragma GCC diagnostic pop
private:
typedef typename __gnu_cxx::__alloc_traits<_Alloc>::template
rebind<value_type>::other _Pair_alloc_type;
typedef _Rb_tree<key_type, value_type, _Select1st<value_type>,
key_compare, _Pair_alloc_type> _Rep_type;
_Rep_type _M_t;
typedef __gnu_cxx::__alloc_traits<_Pair_alloc_type> _Alloc_traits;
template<typename _Up, typename _Vp = remove_reference_t<_Up>>
static constexpr bool __usable_key
= __or_v<is_same<const _Vp, const _Key>,
__and_<is_scalar<_Vp>, is_scalar<_Key>>>;
public:
typedef typename _Alloc_traits::pointer pointer;
typedef typename _Alloc_traits::const_pointer const_pointer;
typedef typename _Alloc_traits::reference reference;
typedef typename _Alloc_traits::const_reference const_reference;
typedef typename _Rep_type::iterator iterator;
typedef typename _Rep_type::const_iterator const_iterator;
typedef typename _Rep_type::size_type size_type;
typedef typename _Rep_type::difference_type difference_type;
typedef typename _Rep_type::reverse_iterator reverse_iterator;
typedef typename _Rep_type::const_reverse_iterator const_reverse_iterator;
using node_type = typename _Rep_type::node_type;
using insert_return_type = typename _Rep_type::insert_return_type;
map() = default;
explicit
map(const _Compare& __comp,
const allocator_type& __a = allocator_type())
: _M_t(__comp, _Pair_alloc_type(__a)) { }
map(const map&) = default;
map(map&&) = default;
map(initializer_list<value_type> __l,
const _Compare& __comp = _Compare(),
const allocator_type& __a = allocator_type())
: _M_t(__comp, _Pair_alloc_type(__a))
{ _M_t._M_insert_range_unique(__l.begin(), __l.end()); }
explicit
map(const allocator_type& __a)
: _M_t(_Pair_alloc_type(__a)) { }
map(const map& __m, const __type_identity_t<allocator_type>& __a)
: _M_t(__m._M_t, _Pair_alloc_type(__a)) { }
map(map&& __m, const __type_identity_t<allocator_type>& __a)
noexcept(is_nothrow_copy_constructible<_Compare>::value
&& _Alloc_traits::_S_always_equal())
: _M_t(std::move(__m._M_t), _Pair_alloc_type(__a)) { }
map(initializer_list<value_type> __l, const allocator_type& __a)
: _M_t(_Pair_alloc_type(__a))
{ _M_t._M_insert_range_unique(__l.begin(), __l.end()); }
template<typename _InputIterator>
map(_InputIterator __first, _InputIterator __last,
const allocator_type& __a)
: _M_t(_Pair_alloc_type(__a))
{ _M_t._M_insert_range_unique(__first, __last); }
template<typename _InputIterator>
map(_InputIterator __first, _InputIterator __last)
: _M_t()
{ _M_t._M_insert_range_unique(__first, __last); }
template<typename _InputIterator>
map(_InputIterator __first, _InputIterator __last,
const _Compare& __comp,
const allocator_type& __a = allocator_type())
: _M_t(__comp, _Pair_alloc_type(__a))
{ _M_t._M_insert_range_unique(__first, __last); }
~map() = default;
map&
operator=(const map&) = default;
map&
operator=(map&&) = default;
map&
operator=(initializer_list<value_type> __l)
{
_M_t._M_assign_unique(__l.begin(), __l.end());
return *this;
}
allocator_type
get_allocator() const noexcept
{ return allocator_type(_M_t.get_allocator()); }
iterator
begin() noexcept
{ return _M_t.begin(); }
const_iterator
begin() const noexcept
{ return _M_t.begin(); }
iterator
end() noexcept
{ return _M_t.end(); }
const_iterator
end() const noexcept
{ return _M_t.end(); }
reverse_iterator
rbegin() noexcept
{ return _M_t.rbegin(); }
const_reverse_iterator
rbegin() const noexcept
{ return _M_t.rbegin(); }
reverse_iterator
rend() noexcept
{ return _M_t.rend(); }
const_reverse_iterator
rend() const noexcept
{ return _M_t.rend(); }
const_iterator
cbegin() const noexcept
{ return _M_t.begin(); }
const_iterator
cend() const noexcept
{ return _M_t.end(); }
const_reverse_iterator
crbegin() const noexcept
{ return _M_t.rbegin(); }
const_reverse_iterator
crend() const noexcept
{ return _M_t.rend(); }
[[__nodiscard__]] bool
empty() const noexcept
{ return _M_t.empty(); }
size_type
size() const noexcept
{ return _M_t.size(); }
size_type
max_size() const noexcept
{ return _M_t.max_size(); }
mapped_type&
operator[](const key_type& __k)
{
iterator __i = lower_bound(__k);
if (__i == end() || key_comp()(__k, (*__i).first))
__i = _M_t._M_emplace_hint_unique(__i, std::piecewise_construct,
std::tuple<const key_type&>(__k),
std::tuple<>());
return (*__i).second;
}
mapped_type&
operator[](key_type&& __k)
{
iterator __i = lower_bound(__k);
if (__i == end() || key_comp()(__k, (*__i).first))
__i = _M_t._M_emplace_hint_unique(__i, std::piecewise_construct,
std::forward_as_tuple(std::move(__k)),
std::tuple<>());
return (*__i).second;
}
mapped_type&
at(const key_type& __k)
{
iterator __i = lower_bound(__k);
if (__i == end() || key_comp()(__k, (*__i).first))
__throw_out_of_range(("map::at"));
return (*__i).second;
}
const mapped_type&
at(const key_type& __k) const
{
const_iterator __i = lower_bound(__k);
if (__i == end() || key_comp()(__k, (*__i).first))
__throw_out_of_range(("map::at"));
return (*__i).second;
}
template<typename... _Args>
std::pair<iterator, bool>
emplace(_Args&&... __args)
{
if constexpr (sizeof...(_Args) == 2)
if constexpr (is_same_v<allocator_type, allocator<value_type>>)
{
auto&& [__a, __v] = pair<_Args&...>(__args...);
if constexpr (__usable_key<decltype(__a)>)
{
const key_type& __k = __a;
iterator __i = lower_bound(__k);
if (__i == end() || key_comp()(__k, (*__i).first))
{
__i = emplace_hint(__i, std::forward<_Args>(__args)...);
return {__i, true};
}
return {__i, false};
}
}
return _M_t._M_emplace_unique(std::forward<_Args>(__args)...);
}
template<typename... _Args>
iterator
emplace_hint(const_iterator __pos, _Args&&... __args)
{
return _M_t._M_emplace_hint_unique(__pos,
std::forward<_Args>(__args)...);
}
node_type
extract(const_iterator __pos)
{
do { if (std::__is_constant_evaluated() && !bool(__pos != end())) __builtin_unreachable(); } while (false);
return _M_t.extract(__pos);
}
node_type
extract(const key_type& __x)
{ return _M_t.extract(__x); }
insert_return_type
insert(node_type&& __nh)
{ return _M_t._M_reinsert_node_unique(std::move(__nh)); }
iterator
insert(const_iterator __hint, node_type&& __nh)
{ return _M_t._M_reinsert_node_hint_unique(__hint, std::move(__nh)); }
template<typename, typename>
friend struct std::_Rb_tree_merge_helper;
template<typename _Cmp2>
void
merge(map<_Key, _Tp, _Cmp2, _Alloc>& __source)
{
using _Merge_helper = _Rb_tree_merge_helper<map, _Cmp2>;
_M_t._M_merge_unique(_Merge_helper::_S_get_tree(__source));
}
template<typename _Cmp2>
void
merge(map<_Key, _Tp, _Cmp2, _Alloc>&& __source)
{ merge(__source); }
template<typename _Cmp2>
void
merge(multimap<_Key, _Tp, _Cmp2, _Alloc>& __source)
{
using _Merge_helper = _Rb_tree_merge_helper<map, _Cmp2>;
_M_t._M_merge_unique(_Merge_helper::_S_get_tree(__source));
}
template<typename _Cmp2>
void
merge(multimap<_Key, _Tp, _Cmp2, _Alloc>&& __source)
{ merge(__source); }
template <typename... _Args>
pair<iterator, bool>
try_emplace(const key_type& __k, _Args&&... __args)
{
iterator __i = lower_bound(__k);
if (__i == end() || key_comp()(__k, (*__i).first))
{
__i = emplace_hint(__i, std::piecewise_construct,
std::forward_as_tuple(__k),
std::forward_as_tuple(
std::forward<_Args>(__args)...));
return {__i, true};
}
return {__i, false};
}
template <typename... _Args>
pair<iterator, bool>
try_emplace(key_type&& __k, _Args&&... __args)
{
iterator __i = lower_bound(__k);
if (__i == end() || key_comp()(__k, (*__i).first))
{
__i = emplace_hint(__i, std::piecewise_construct,
std::forward_as_tuple(std::move(__k)),
std::forward_as_tuple(
std::forward<_Args>(__args)...));
return {__i, true};
}
return {__i, false};
}
template <typename... _Args>
iterator
try_emplace(const_iterator __hint, const key_type& __k,
_Args&&... __args)
{
iterator __i;
auto __true_hint = _M_t._M_get_insert_hint_unique_pos(__hint, __k);
if (__true_hint.second)
__i = emplace_hint(iterator(__true_hint.second),
std::piecewise_construct,
std::forward_as_tuple(__k),
std::forward_as_tuple(
std::forward<_Args>(__args)...));
else
__i = iterator(__true_hint.first);
return __i;
}
template <typename... _Args>
iterator
try_emplace(const_iterator __hint, key_type&& __k, _Args&&... __args)
{
iterator __i;
auto __true_hint = _M_t._M_get_insert_hint_unique_pos(__hint, __k);
if (__true_hint.second)
__i = emplace_hint(iterator(__true_hint.second),
std::piecewise_construct,
std::forward_as_tuple(std::move(__k)),
std::forward_as_tuple(
std::forward<_Args>(__args)...));
else
__i = iterator(__true_hint.first);
return __i;
}
std::pair<iterator, bool>
insert(const value_type& __x)
{ return _M_t._M_insert_unique(__x); }
std::pair<iterator, bool>
insert(value_type&& __x)
{ return _M_t._M_insert_unique(std::move(__x)); }
template<typename _Pair>
__enable_if_t<is_constructible<value_type, _Pair>::value,
pair<iterator, bool>>
insert(_Pair&& __x)
{
using _P2 = remove_reference_t<_Pair>;
if constexpr (__is_pair<remove_const_t<_P2>>)
if constexpr (is_same_v<allocator_type, allocator<value_type>>)
if constexpr (__usable_key<typename _P2::first_type>)
{
const key_type& __k = __x.first;
iterator __i = lower_bound(__k);
if (__i == end() || key_comp()(__k, (*__i).first))
{
__i = emplace_hint(__i, std::forward<_Pair>(__x));
return {__i, true};
}
return {__i, false};
}
return _M_t._M_emplace_unique(std::forward<_Pair>(__x));
}
void
insert(std::initializer_list<value_type> __list)
{ insert(__list.begin(), __list.end()); }
iterator
insert(const_iterator __position, const value_type& __x)
{ return _M_t._M_insert_unique_(__position, __x); }
iterator
insert(const_iterator __position, value_type&& __x)
{ return _M_t._M_insert_unique_(__position, std::move(__x)); }
template<typename _Pair>
__enable_if_t<is_constructible<value_type, _Pair>::value, iterator>
insert(const_iterator __position, _Pair&& __x)
{
return _M_t._M_emplace_hint_unique(__position,
std::forward<_Pair>(__x));
}
template<typename _InputIterator>
void
insert(_InputIterator __first, _InputIterator __last)
{ _M_t._M_insert_range_unique(__first, __last); }
template <typename _Obj>
pair<iterator, bool>
insert_or_assign(const key_type& __k, _Obj&& __obj)
{
iterator __i = lower_bound(__k);
if (__i == end() || key_comp()(__k, (*__i).first))
{
__i = emplace_hint(__i, std::piecewise_construct,
std::forward_as_tuple(__k),
std::forward_as_tuple(
std::forward<_Obj>(__obj)));
return {__i, true};
}
(*__i).second = std::forward<_Obj>(__obj);
return {__i, false};
}
template <typename _Obj>
pair<iterator, bool>
insert_or_assign(key_type&& __k, _Obj&& __obj)
{
iterator __i = lower_bound(__k);
if (__i == end() || key_comp()(__k, (*__i).first))
{
__i = emplace_hint(__i, std::piecewise_construct,
std::forward_as_tuple(std::move(__k)),
std::forward_as_tuple(
std::forward<_Obj>(__obj)));
return {__i, true};
}
(*__i).second = std::forward<_Obj>(__obj);
return {__i, false};
}
template <typename _Obj>
iterator
insert_or_assign(const_iterator __hint,
const key_type& __k, _Obj&& __obj)
{
iterator __i;
auto __true_hint = _M_t._M_get_insert_hint_unique_pos(__hint, __k);
if (__true_hint.second)
{
return emplace_hint(iterator(__true_hint.second),
std::piecewise_construct,
std::forward_as_tuple(__k),
std::forward_as_tuple(
std::forward<_Obj>(__obj)));
}
__i = iterator(__true_hint.first);
(*__i).second = std::forward<_Obj>(__obj);
return __i;
}
template <typename _Obj>
iterator
insert_or_assign(const_iterator __hint, key_type&& __k, _Obj&& __obj)
{
iterator __i;
auto __true_hint = _M_t._M_get_insert_hint_unique_pos(__hint, __k);
if (__true_hint.second)
{
return emplace_hint(iterator(__true_hint.second),
std::piecewise_construct,
std::forward_as_tuple(std::move(__k)),
std::forward_as_tuple(
std::forward<_Obj>(__obj)));
}
__i = iterator(__true_hint.first);
(*__i).second = std::forward<_Obj>(__obj);
return __i;
}
iterator
erase(const_iterator __position)
{ return _M_t.erase(__position); }
__attribute ((__abi_tag__ ("cxx11")))
iterator
erase(iterator __position)
{ return _M_t.erase(__position); }
size_type
erase(const key_type& __x)
{ return _M_t.erase(__x); }
iterator
erase(const_iterator __first, const_iterator __last)
{ return _M_t.erase(__first, __last); }
void
swap(map& __x)
noexcept(__is_nothrow_swappable<_Compare>::value)
{ _M_t.swap(__x._M_t); }
void
clear() noexcept
{ _M_t.clear(); }
key_compare
key_comp() const
{ return _M_t.key_comp(); }
value_compare
value_comp() const
{ return value_compare(_M_t.key_comp()); }
iterator
find(const key_type& __x)
{ return _M_t.find(__x); }
template<typename _Kt>
auto
find(const _Kt& __x) -> decltype(_M_t._M_find_tr(__x))
{ return _M_t._M_find_tr(__x); }
const_iterator
find(const key_type& __x) const
{ return _M_t.find(__x); }
template<typename _Kt>
auto
find(const _Kt& __x) const -> decltype(_M_t._M_find_tr(__x))
{ return _M_t._M_find_tr(__x); }
size_type
count(const key_type& __x) const
{ return _M_t.find(__x) == _M_t.end() ? 0 : 1; }
template<typename _Kt>
auto
count(const _Kt& __x) const -> decltype(_M_t._M_count_tr(__x))
{ return _M_t._M_count_tr(__x); }
iterator
lower_bound(const key_type& __x)
{ return _M_t.lower_bound(__x); }
template<typename _Kt>
auto
lower_bound(const _Kt& __x)
-> decltype(iterator(_M_t._M_lower_bound_tr(__x)))
{ return iterator(_M_t._M_lower_bound_tr(__x)); }
const_iterator
lower_bound(const key_type& __x) const
{ return _M_t.lower_bound(__x); }
template<typename _Kt>
auto
lower_bound(const _Kt& __x) const
-> decltype(const_iterator(_M_t._M_lower_bound_tr(__x)))
{ return const_iterator(_M_t._M_lower_bound_tr(__x)); }
iterator
upper_bound(const key_type& __x)
{ return _M_t.upper_bound(__x); }
template<typename _Kt>
auto
upper_bound(const _Kt& __x)
-> decltype(iterator(_M_t._M_upper_bound_tr(__x)))
{ return iterator(_M_t._M_upper_bound_tr(__x)); }
const_iterator
upper_bound(const key_type& __x) const
{ return _M_t.upper_bound(__x); }
template<typename _Kt>
auto
upper_bound(const _Kt& __x) const
-> decltype(const_iterator(_M_t._M_upper_bound_tr(__x)))
{ return const_iterator(_M_t._M_upper_bound_tr(__x)); }
std::pair<iterator, iterator>
equal_range(const key_type& __x)
{ return _M_t.equal_range(__x); }
template<typename _Kt>
auto
equal_range(const _Kt& __x)
-> decltype(pair<iterator, iterator>(_M_t._M_equal_range_tr(__x)))
{ return pair<iterator, iterator>(_M_t._M_equal_range_tr(__x)); }
std::pair<const_iterator, const_iterator>
equal_range(const key_type& __x) const
{ return _M_t.equal_range(__x); }
template<typename _Kt>
auto
equal_range(const _Kt& __x) const
-> decltype(pair<const_iterator, const_iterator>(
_M_t._M_equal_range_tr(__x)))
{
return pair<const_iterator, const_iterator>(
_M_t._M_equal_range_tr(__x));
}
template<typename _K1, typename _T1, typename _C1, typename _A1>
friend bool
operator==(const map<_K1, _T1, _C1, _A1>&,
const map<_K1, _T1, _C1, _A1>&);
template<typename _K1, typename _T1, typename _C1, typename _A1>
friend bool
operator<(const map<_K1, _T1, _C1, _A1>&,
const map<_K1, _T1, _C1, _A1>&);
};
template<typename _InputIterator,
typename _Compare = less<__iter_key_t<_InputIterator>>,
typename _Allocator = allocator<__iter_to_alloc_t<_InputIterator>>,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireNotAllocator<_Compare>,
typename = _RequireAllocator<_Allocator>>
map(_InputIterator, _InputIterator,
_Compare = _Compare(), _Allocator = _Allocator())
-> map<__iter_key_t<_InputIterator>, __iter_val_t<_InputIterator>,
_Compare, _Allocator>;
template<typename _Key, typename _Tp, typename _Compare = less<_Key>,
typename _Allocator = allocator<pair<const _Key, _Tp>>,
typename = _RequireNotAllocator<_Compare>,
typename = _RequireAllocator<_Allocator>>
map(initializer_list<pair<_Key, _Tp>>,
_Compare = _Compare(), _Allocator = _Allocator())
-> map<_Key, _Tp, _Compare, _Allocator>;
template <typename _InputIterator, typename _Allocator,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireAllocator<_Allocator>>
map(_InputIterator, _InputIterator, _Allocator)
-> map<__iter_key_t<_InputIterator>, __iter_val_t<_InputIterator>,
less<__iter_key_t<_InputIterator>>, _Allocator>;
template<typename _Key, typename _Tp, typename _Allocator,
typename = _RequireAllocator<_Allocator>>
map(initializer_list<pair<_Key, _Tp>>, _Allocator)
-> map<_Key, _Tp, less<_Key>, _Allocator>;
template<typename _Key, typename _Tp, typename _Compare, typename _Alloc>
inline bool
operator==(const map<_Key, _Tp, _Compare, _Alloc>& __x,
const map<_Key, _Tp, _Compare, _Alloc>& __y)
{ return __x._M_t == __y._M_t; }
template<typename _Key, typename _Tp, typename _Compare, typename _Alloc>
inline bool
operator<(const map<_Key, _Tp, _Compare, _Alloc>& __x,
const map<_Key, _Tp, _Compare, _Alloc>& __y)
{ return __x._M_t < __y._M_t; }
template<typename _Key, typename _Tp, typename _Compare, typename _Alloc>
inline bool
operator!=(const map<_Key, _Tp, _Compare, _Alloc>& __x,
const map<_Key, _Tp, _Compare, _Alloc>& __y)
{ return !(__x == __y); }
template<typename _Key, typename _Tp, typename _Compare, typename _Alloc>
inline bool
operator>(const map<_Key, _Tp, _Compare, _Alloc>& __x,
const map<_Key, _Tp, _Compare, _Alloc>& __y)
{ return __y < __x; }
template<typename _Key, typename _Tp, typename _Compare, typename _Alloc>
inline bool
operator<=(const map<_Key, _Tp, _Compare, _Alloc>& __x,
const map<_Key, _Tp, _Compare, _Alloc>& __y)
{ return !(__y < __x); }
template<typename _Key, typename _Tp, typename _Compare, typename _Alloc>
inline bool
operator>=(const map<_Key, _Tp, _Compare, _Alloc>& __x,
const map<_Key, _Tp, _Compare, _Alloc>& __y)
{ return !(__x < __y); }
template<typename _Key, typename _Tp, typename _Compare, typename _Alloc>
inline void
swap(map<_Key, _Tp, _Compare, _Alloc>& __x,
map<_Key, _Tp, _Compare, _Alloc>& __y)
noexcept(noexcept(__x.swap(__y)))
{ __x.swap(__y); }
template<typename _Key, typename _Val, typename _Cmp1, typename _Alloc,
typename _Cmp2>
struct
_Rb_tree_merge_helper<std::map<_Key, _Val, _Cmp1, _Alloc>,
_Cmp2>
{
private:
friend class std::map<_Key, _Val, _Cmp1, _Alloc>;
static auto&
_S_get_tree(std::map<_Key, _Val, _Cmp2, _Alloc>& __map)
{ return __map._M_t; }
static auto&
_S_get_tree(std::multimap<_Key, _Val, _Cmp2, _Alloc>& __map)
{ return __map._M_t; }
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
template <typename _Key, typename _Tp, typename _Compare, typename _Alloc>
class map;
template <typename _Key, typename _Tp,
typename _Compare = std::less<_Key>,
typename _Alloc = std::allocator<std::pair<const _Key, _Tp> > >
class multimap
{
public:
typedef _Key key_type;
typedef _Tp mapped_type;
typedef std::pair<const _Key, _Tp> value_type;
typedef _Compare key_compare;
typedef _Alloc allocator_type;
private:
public:
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
class value_compare
: public std::binary_function<value_type, value_type, bool>
{
friend class multimap<_Key, _Tp, _Compare, _Alloc>;
protected:
_Compare comp;
value_compare(_Compare __c)
: comp(__c) { }
public:
bool operator()(const value_type& __x, const value_type& __y) const
{ return comp(__x.first, __y.first); }
};
#pragma GCC diagnostic pop
private:
typedef typename __gnu_cxx::__alloc_traits<_Alloc>::template
rebind<value_type>::other _Pair_alloc_type;
typedef _Rb_tree<key_type, value_type, _Select1st<value_type>,
key_compare, _Pair_alloc_type> _Rep_type;
_Rep_type _M_t;
typedef __gnu_cxx::__alloc_traits<_Pair_alloc_type> _Alloc_traits;
public:
typedef typename _Alloc_traits::pointer pointer;
typedef typename _Alloc_traits::const_pointer const_pointer;
typedef typename _Alloc_traits::reference reference;
typedef typename _Alloc_traits::const_reference const_reference;
typedef typename _Rep_type::iterator iterator;
typedef typename _Rep_type::const_iterator const_iterator;
typedef typename _Rep_type::size_type size_type;
typedef typename _Rep_type::difference_type difference_type;
typedef typename _Rep_type::reverse_iterator reverse_iterator;
typedef typename _Rep_type::const_reverse_iterator const_reverse_iterator;
using node_type = typename _Rep_type::node_type;
multimap() = default;
explicit
multimap(const _Compare& __comp,
const allocator_type& __a = allocator_type())
: _M_t(__comp, _Pair_alloc_type(__a)) { }
multimap(const multimap&) = default;
multimap(multimap&&) = default;
multimap(initializer_list<value_type> __l,
const _Compare& __comp = _Compare(),
const allocator_type& __a = allocator_type())
: _M_t(__comp, _Pair_alloc_type(__a))
{ _M_t._M_insert_range_equal(__l.begin(), __l.end()); }
explicit
multimap(const allocator_type& __a)
: _M_t(_Pair_alloc_type(__a)) { }
multimap(const multimap& __m,
const __type_identity_t<allocator_type>& __a)
: _M_t(__m._M_t, _Pair_alloc_type(__a)) { }
multimap(multimap&& __m, const __type_identity_t<allocator_type>& __a)
noexcept(is_nothrow_copy_constructible<_Compare>::value
&& _Alloc_traits::_S_always_equal())
: _M_t(std::move(__m._M_t), _Pair_alloc_type(__a)) { }
multimap(initializer_list<value_type> __l, const allocator_type& __a)
: _M_t(_Pair_alloc_type(__a))
{ _M_t._M_insert_range_equal(__l.begin(), __l.end()); }
template<typename _InputIterator>
multimap(_InputIterator __first, _InputIterator __last,
const allocator_type& __a)
: _M_t(_Pair_alloc_type(__a))
{ _M_t._M_insert_range_equal(__first, __last); }
template<typename _InputIterator>
multimap(_InputIterator __first, _InputIterator __last)
: _M_t()
{ _M_t._M_insert_range_equal(__first, __last); }
template<typename _InputIterator>
multimap(_InputIterator __first, _InputIterator __last,
const _Compare& __comp,
const allocator_type& __a = allocator_type())
: _M_t(__comp, _Pair_alloc_type(__a))
{ _M_t._M_insert_range_equal(__first, __last); }
~multimap() = default;
multimap&
operator=(const multimap&) = default;
multimap&
operator=(multimap&&) = default;
multimap&
operator=(initializer_list<value_type> __l)
{
_M_t._M_assign_equal(__l.begin(), __l.end());
return *this;
}
allocator_type
get_allocator() const noexcept
{ return allocator_type(_M_t.get_allocator()); }
iterator
begin() noexcept
{ return _M_t.begin(); }
const_iterator
begin() const noexcept
{ return _M_t.begin(); }
iterator
end() noexcept
{ return _M_t.end(); }
const_iterator
end() const noexcept
{ return _M_t.end(); }
reverse_iterator
rbegin() noexcept
{ return _M_t.rbegin(); }
const_reverse_iterator
rbegin() const noexcept
{ return _M_t.rbegin(); }
reverse_iterator
rend() noexcept
{ return _M_t.rend(); }
const_reverse_iterator
rend() const noexcept
{ return _M_t.rend(); }
const_iterator
cbegin() const noexcept
{ return _M_t.begin(); }
const_iterator
cend() const noexcept
{ return _M_t.end(); }
const_reverse_iterator
crbegin() const noexcept
{ return _M_t.rbegin(); }
const_reverse_iterator
crend() const noexcept
{ return _M_t.rend(); }
[[__nodiscard__]] bool
empty() const noexcept
{ return _M_t.empty(); }
size_type
size() const noexcept
{ return _M_t.size(); }
size_type
max_size() const noexcept
{ return _M_t.max_size(); }
template<typename... _Args>
iterator
emplace(_Args&&... __args)
{ return _M_t._M_emplace_equal(std::forward<_Args>(__args)...); }
template<typename... _Args>
iterator
emplace_hint(const_iterator __pos, _Args&&... __args)
{
return _M_t._M_emplace_hint_equal(__pos,
std::forward<_Args>(__args)...);
}
iterator
insert(const value_type& __x)
{ return _M_t._M_insert_equal(__x); }
iterator
insert(value_type&& __x)
{ return _M_t._M_insert_equal(std::move(__x)); }
template<typename _Pair>
__enable_if_t<is_constructible<value_type, _Pair>::value, iterator>
insert(_Pair&& __x)
{ return _M_t._M_emplace_equal(std::forward<_Pair>(__x)); }
iterator
insert(const_iterator __position, const value_type& __x)
{ return _M_t._M_insert_equal_(__position, __x); }
iterator
insert(const_iterator __position, value_type&& __x)
{ return _M_t._M_insert_equal_(__position, std::move(__x)); }
template<typename _Pair>
__enable_if_t<is_constructible<value_type, _Pair&&>::value, iterator>
insert(const_iterator __position, _Pair&& __x)
{
return _M_t._M_emplace_hint_equal(__position,
std::forward<_Pair>(__x));
}
template<typename _InputIterator>
void
insert(_InputIterator __first, _InputIterator __last)
{ _M_t._M_insert_range_equal(__first, __last); }
void
insert(initializer_list<value_type> __l)
{ this->insert(__l.begin(), __l.end()); }
node_type
extract(const_iterator __pos)
{
do { if (std::__is_constant_evaluated() && !bool(__pos != end())) __builtin_unreachable(); } while (false);
return _M_t.extract(__pos);
}
node_type
extract(const key_type& __x)
{ return _M_t.extract(__x); }
iterator
insert(node_type&& __nh)
{ return _M_t._M_reinsert_node_equal(std::move(__nh)); }
iterator
insert(const_iterator __hint, node_type&& __nh)
{ return _M_t._M_reinsert_node_hint_equal(__hint, std::move(__nh)); }
template<typename, typename>
friend struct std::_Rb_tree_merge_helper;
template<typename _Cmp2>
void
merge(multimap<_Key, _Tp, _Cmp2, _Alloc>& __source)
{
using _Merge_helper = _Rb_tree_merge_helper<multimap, _Cmp2>;
_M_t._M_merge_equal(_Merge_helper::_S_get_tree(__source));
}
template<typename _Cmp2>
void
merge(multimap<_Key, _Tp, _Cmp2, _Alloc>&& __source)
{ merge(__source); }
template<typename _Cmp2>
void
merge(map<_Key, _Tp, _Cmp2, _Alloc>& __source)
{
using _Merge_helper = _Rb_tree_merge_helper<multimap, _Cmp2>;
_M_t._M_merge_equal(_Merge_helper::_S_get_tree(__source));
}
template<typename _Cmp2>
void
merge(map<_Key, _Tp, _Cmp2, _Alloc>&& __source)
{ merge(__source); }
iterator
erase(const_iterator __position)
{ return _M_t.erase(__position); }
__attribute ((__abi_tag__ ("cxx11")))
iterator
erase(iterator __position)
{ return _M_t.erase(__position); }
size_type
erase(const key_type& __x)
{ return _M_t.erase(__x); }
iterator
erase(const_iterator __first, const_iterator __last)
{ return _M_t.erase(__first, __last); }
void
swap(multimap& __x)
noexcept(__is_nothrow_swappable<_Compare>::value)
{ _M_t.swap(__x._M_t); }
void
clear() noexcept
{ _M_t.clear(); }
key_compare
key_comp() const
{ return _M_t.key_comp(); }
value_compare
value_comp() const
{ return value_compare(_M_t.key_comp()); }
iterator
find(const key_type& __x)
{ return _M_t.find(__x); }
template<typename _Kt>
auto
find(const _Kt& __x) -> decltype(_M_t._M_find_tr(__x))
{ return _M_t._M_find_tr(__x); }
const_iterator
find(const key_type& __x) const
{ return _M_t.find(__x); }
template<typename _Kt>
auto
find(const _Kt& __x) const -> decltype(_M_t._M_find_tr(__x))
{ return _M_t._M_find_tr(__x); }
size_type
count(const key_type& __x) const
{ return _M_t.count(__x); }
template<typename _Kt>
auto
count(const _Kt& __x) const -> decltype(_M_t._M_count_tr(__x))
{ return _M_t._M_count_tr(__x); }
iterator
lower_bound(const key_type& __x)
{ return _M_t.lower_bound(__x); }
template<typename _Kt>
auto
lower_bound(const _Kt& __x)
-> decltype(iterator(_M_t._M_lower_bound_tr(__x)))
{ return iterator(_M_t._M_lower_bound_tr(__x)); }
const_iterator
lower_bound(const key_type& __x) const
{ return _M_t.lower_bound(__x); }
template<typename _Kt>
auto
lower_bound(const _Kt& __x) const
-> decltype(const_iterator(_M_t._M_lower_bound_tr(__x)))
{ return const_iterator(_M_t._M_lower_bound_tr(__x)); }
iterator
upper_bound(const key_type& __x)
{ return _M_t.upper_bound(__x); }
template<typename _Kt>
auto
upper_bound(const _Kt& __x)
-> decltype(iterator(_M_t._M_upper_bound_tr(__x)))
{ return iterator(_M_t._M_upper_bound_tr(__x)); }
const_iterator
upper_bound(const key_type& __x) const
{ return _M_t.upper_bound(__x); }
template<typename _Kt>
auto
upper_bound(const _Kt& __x) const
-> decltype(const_iterator(_M_t._M_upper_bound_tr(__x)))
{ return const_iterator(_M_t._M_upper_bound_tr(__x)); }
std::pair<iterator, iterator>
equal_range(const key_type& __x)
{ return _M_t.equal_range(__x); }
template<typename _Kt>
auto
equal_range(const _Kt& __x)
-> decltype(pair<iterator, iterator>(_M_t._M_equal_range_tr(__x)))
{ return pair<iterator, iterator>(_M_t._M_equal_range_tr(__x)); }
std::pair<const_iterator, const_iterator>
equal_range(const key_type& __x) const
{ return _M_t.equal_range(__x); }
template<typename _Kt>
auto
equal_range(const _Kt& __x) const
-> decltype(pair<const_iterator, const_iterator>(
_M_t._M_equal_range_tr(__x)))
{
return pair<const_iterator, const_iterator>(
_M_t._M_equal_range_tr(__x));
}
template<typename _K1, typename _T1, typename _C1, typename _A1>
friend bool
operator==(const multimap<_K1, _T1, _C1, _A1>&,
const multimap<_K1, _T1, _C1, _A1>&);
template<typename _K1, typename _T1, typename _C1, typename _A1>
friend bool
operator<(const multimap<_K1, _T1, _C1, _A1>&,
const multimap<_K1, _T1, _C1, _A1>&);
};
template<typename _InputIterator,
typename _Compare = less<__iter_key_t<_InputIterator>>,
typename _Allocator = allocator<__iter_to_alloc_t<_InputIterator>>,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireNotAllocator<_Compare>,
typename = _RequireAllocator<_Allocator>>
multimap(_InputIterator, _InputIterator,
_Compare = _Compare(), _Allocator = _Allocator())
-> multimap<__iter_key_t<_InputIterator>, __iter_val_t<_InputIterator>,
_Compare, _Allocator>;
template<typename _Key, typename _Tp, typename _Compare = less<_Key>,
typename _Allocator = allocator<pair<const _Key, _Tp>>,
typename = _RequireNotAllocator<_Compare>,
typename = _RequireAllocator<_Allocator>>
multimap(initializer_list<pair<_Key, _Tp>>,
_Compare = _Compare(), _Allocator = _Allocator())
-> multimap<_Key, _Tp, _Compare, _Allocator>;
template<typename _InputIterator, typename _Allocator,
typename = _RequireInputIter<_InputIterator>,
typename = _RequireAllocator<_Allocator>>
multimap(_InputIterator, _InputIterator, _Allocator)
-> multimap<__iter_key_t<_InputIterator>, __iter_val_t<_InputIterator>,
less<__iter_key_t<_InputIterator>>, _Allocator>;
template<typename _Key, typename _Tp, typename _Allocator,
typename = _RequireAllocator<_Allocator>>
multimap(initializer_list<pair<_Key, _Tp>>, _Allocator)
-> multimap<_Key, _Tp, less<_Key>, _Allocator>;
template<typename _Key, typename _Tp, typename _Compare, typename _Alloc>
inline bool
operator==(const multimap<_Key, _Tp, _Compare, _Alloc>& __x,
const multimap<_Key, _Tp, _Compare, _Alloc>& __y)
{ return __x._M_t == __y._M_t; }
template<typename _Key, typename _Tp, typename _Compare, typename _Alloc>
inline bool
operator<(const multimap<_Key, _Tp, _Compare, _Alloc>& __x,
const multimap<_Key, _Tp, _Compare, _Alloc>& __y)
{ return __x._M_t < __y._M_t; }
template<typename _Key, typename _Tp, typename _Compare, typename _Alloc>
inline bool
operator!=(const multimap<_Key, _Tp, _Compare, _Alloc>& __x,
const multimap<_Key, _Tp, _Compare, _Alloc>& __y)
{ return !(__x == __y); }
template<typename _Key, typename _Tp, typename _Compare, typename _Alloc>
inline bool
operator>(const multimap<_Key, _Tp, _Compare, _Alloc>& __x,
const multimap<_Key, _Tp, _Compare, _Alloc>& __y)
{ return __y < __x; }
template<typename _Key, typename _Tp, typename _Compare, typename _Alloc>
inline bool
operator<=(const multimap<_Key, _Tp, _Compare, _Alloc>& __x,
const multimap<_Key, _Tp, _Compare, _Alloc>& __y)
{ return !(__y < __x); }
template<typename _Key, typename _Tp, typename _Compare, typename _Alloc>
inline bool
operator>=(const multimap<_Key, _Tp, _Compare, _Alloc>& __x,
const multimap<_Key, _Tp, _Compare, _Alloc>& __y)
{ return !(__x < __y); }
template<typename _Key, typename _Tp, typename _Compare, typename _Alloc>
inline void
swap(multimap<_Key, _Tp, _Compare, _Alloc>& __x,
multimap<_Key, _Tp, _Compare, _Alloc>& __y)
noexcept(noexcept(__x.swap(__y)))
{ __x.swap(__y); }
template<typename _Key, typename _Val, typename _Cmp1, typename _Alloc,
typename _Cmp2>
struct
_Rb_tree_merge_helper<std::multimap<_Key, _Val, _Cmp1, _Alloc>,
_Cmp2>
{
private:
friend class std::multimap<_Key, _Val, _Cmp1, _Alloc>;
static auto&
_S_get_tree(std::map<_Key, _Val, _Cmp2, _Alloc>& __map)
{ return __map._M_t; }
static auto&
_S_get_tree(std::multimap<_Key, _Val, _Cmp2, _Alloc>& __map)
{ return __map._M_t; }
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
namespace pmr
{
template<typename _Key, typename _Tp, typename _Cmp = std::less<_Key>>
using map
= std::map<_Key, _Tp, _Cmp,
polymorphic_allocator<pair<const _Key, _Tp>>>;
template<typename _Key, typename _Tp, typename _Cmp = std::less<_Key>>
using multimap
= std::multimap<_Key, _Tp, _Cmp,
polymorphic_allocator<pair<const _Key, _Tp>>>;
}
}
namespace testing {
namespace internal {
class __attribute__((visibility("default"))) String {
public:
static const char* CloneCString(const char* c_str);
static bool CStringEquals(const char* lhs, const char* rhs);
static std::string ShowWideCString(const wchar_t* wide_c_str);
static bool WideCStringEquals(const wchar_t* lhs, const wchar_t* rhs);
static bool CaseInsensitiveCStringEquals(const char* lhs, const char* rhs);
static bool CaseInsensitiveWideCStringEquals(const wchar_t* lhs,
const wchar_t* rhs);
static bool EndsWithCaseInsensitive(const std::string& str,
const std::string& suffix);
static std::string FormatIntWidth2(int value);
static std::string FormatIntWidthN(int value, int width);
static std::string FormatHexInt(int value);
static std::string FormatHexUInt32(uint32_t value);
static std::string FormatByte(unsigned char value);
private:
String();
};
__attribute__((visibility("default"))) std::string StringStreamToString(::std::stringstream* stream);
}
}
namespace testing {
namespace internal {
class __attribute__((visibility("default"))) FilePath {
public:
FilePath() : pathname_("") {}
FilePath(const FilePath& rhs) : pathname_(rhs.pathname_) {}
explicit FilePath(const std::string& pathname) : pathname_(pathname) {
Normalize();
}
FilePath& operator=(const FilePath& rhs) {
Set(rhs);
return *this;
}
void Set(const FilePath& rhs) { pathname_ = rhs.pathname_; }
const std::string& string() const { return pathname_; }
const char* c_str() const { return pathname_.c_str(); }
static FilePath GetCurrentDir();
static FilePath MakeFileName(const FilePath& directory,
const FilePath& base_name, int number,
const char* extension);
static FilePath ConcatPaths(const FilePath& directory,
const FilePath& relative_path);
static FilePath GenerateUniqueFileName(const FilePath& directory,
const FilePath& base_name,
const char* extension);
bool IsEmpty() const { return pathname_.empty(); }
FilePath RemoveTrailingPathSeparator() const;
FilePath RemoveDirectoryName() const;
FilePath RemoveFileName() const;
FilePath RemoveExtension(const char* extension) const;
bool CreateDirectoriesRecursively() const;
bool CreateFolder() const;
bool FileOrDirectoryExists() const;
bool DirectoryExists() const;
bool IsDirectory() const;
bool IsRootDirectory() const;
bool IsAbsolutePath() const;
private:
void Normalize();
const char* FindLastPathSeparator() const;
size_t CalculateRootLength() const;
std::string pathname_;
};
}
}
#pragma GCC visibility push(default)
namespace __cxxabiv1
{
extern "C"
{
__extension__ typedef int __guard __attribute__((mode (__DI__)));
typedef void __cxa_vec_ctor_return_type;
typedef void __cxa_cdtor_return_type;
}
}
namespace __cxxabiv1
{
extern "C"
{
typedef __cxa_cdtor_return_type (*__cxa_cdtor_type)(void *);
void*
__cxa_vec_new(size_t __element_count, size_t __element_size,
size_t __padding_size, __cxa_cdtor_type __constructor,
__cxa_cdtor_type __destructor);
void*
__cxa_vec_new2(size_t __element_count, size_t __element_size,
size_t __padding_size, __cxa_cdtor_type __constructor,
__cxa_cdtor_type __destructor, void *(*__alloc) (size_t),
void (*__dealloc) (void*));
void*
__cxa_vec_new3(size_t __element_count, size_t __element_size,
size_t __padding_size, __cxa_cdtor_type __constructor,
__cxa_cdtor_type __destructor, void *(*__alloc) (size_t),
void (*__dealloc) (void*, size_t));
__cxa_vec_ctor_return_type
__cxa_vec_ctor(void* __array_address, size_t __element_count,
size_t __element_size, __cxa_cdtor_type __constructor,
__cxa_cdtor_type __destructor);
__cxa_vec_ctor_return_type
__cxa_vec_cctor(void* __dest_array, void* __src_array,
size_t __element_count, size_t __element_size,
__cxa_cdtor_return_type (*__constructor) (void*, void*),
__cxa_cdtor_type __destructor);
void
__cxa_vec_dtor(void* __array_address, size_t __element_count,
size_t __element_size, __cxa_cdtor_type __destructor);
void
__cxa_vec_cleanup(void* __array_address, size_t __element_count, size_t __s,
__cxa_cdtor_type __destructor) noexcept;
void
__cxa_vec_delete(void* __array_address, size_t __element_size,
size_t __padding_size, __cxa_cdtor_type __destructor);
void
__cxa_vec_delete2(void* __array_address, size_t __element_size,
size_t __padding_size, __cxa_cdtor_type __destructor,
void (*__dealloc) (void*));
void
__cxa_vec_delete3(void* __array_address, size_t __element_size,
size_t __padding_size, __cxa_cdtor_type __destructor,
void (*__dealloc) (void*, size_t));
int
__cxa_guard_acquire(__guard*);
void
__cxa_guard_release(__guard*) noexcept;
void
__cxa_guard_abort(__guard*) noexcept;
int
__cxa_atexit(void ( *)(void*), void*, void*) noexcept;
void
__cxa_finalize(void*);
int
__cxa_thread_atexit(void ( *)(void*), void*, void *) noexcept;
void
__cxa_pure_virtual(void) __attribute__ ((__noreturn__));
void
__cxa_deleted_virtual(void) __attribute__ ((__noreturn__));
void
__cxa_bad_cast() __attribute__((__noreturn__));
void
__cxa_bad_typeid() __attribute__((__noreturn__));
void
__cxa_throw_bad_array_new_length() __attribute__((__noreturn__));
char*
__cxa_demangle(const char* __mangled_name, char* __output_buffer,
size_t* __length, int* __status);
}
}
namespace __cxxabiv1
{
class __fundamental_type_info : public std::type_info
{
public:
explicit
__fundamental_type_info(const char* __n) : std::type_info(__n) { }
virtual
~__fundamental_type_info();
};
class __array_type_info : public std::type_info
{
public:
explicit
__array_type_info(const char* __n) : std::type_info(__n) { }
virtual
~__array_type_info();
};
class __function_type_info : public std::type_info
{
public:
explicit
__function_type_info(const char* __n) : std::type_info(__n) { }
virtual
~__function_type_info();
protected:
virtual bool
__is_function_p() const;
};
class __enum_type_info : public std::type_info
{
public:
explicit
__enum_type_info(const char* __n) : std::type_info(__n) { }
virtual
~__enum_type_info();
};
class __pbase_type_info : public std::type_info
{
public:
unsigned int __flags;
const std::type_info* __pointee;
explicit
__pbase_type_info(const char* __n, int __quals,
const std::type_info* __type)
: std::type_info(__n), __flags(__quals), __pointee(__type)
{ }
virtual
~__pbase_type_info();
enum __masks
{
__const_mask = 0x1,
__volatile_mask = 0x2,
__restrict_mask = 0x4,
__incomplete_mask = 0x8,
__incomplete_class_mask = 0x10,
__transaction_safe_mask = 0x20,
__noexcept_mask = 0x40
};
protected:
__pbase_type_info(const __pbase_type_info&);
__pbase_type_info&
operator=(const __pbase_type_info&);
virtual bool
__do_catch(const std::type_info* __thr_type, void** __thr_obj,
unsigned int __outer) const;
inline virtual bool
__pointer_catch(const __pbase_type_info* __thr_type, void** __thr_obj,
unsigned __outer) const;
};
inline bool __pbase_type_info::
__pointer_catch (const __pbase_type_info *thrown_type,
void **thr_obj,
unsigned outer) const
{
return __pointee->__do_catch (thrown_type->__pointee, thr_obj, outer + 2);
}
class __pointer_type_info : public __pbase_type_info
{
public:
explicit
__pointer_type_info(const char* __n, int __quals,
const std::type_info* __type)
: __pbase_type_info (__n, __quals, __type) { }
virtual
~__pointer_type_info();
protected:
virtual bool
__is_pointer_p() const;
virtual bool
__pointer_catch(const __pbase_type_info* __thr_type, void** __thr_obj,
unsigned __outer) const;
};
class __class_type_info;
class __pointer_to_member_type_info : public __pbase_type_info
{
public:
__class_type_info* __context;
explicit
__pointer_to_member_type_info(const char* __n, int __quals,
const std::type_info* __type,
__class_type_info* __klass)
: __pbase_type_info(__n, __quals, __type), __context(__klass) { }
virtual
~__pointer_to_member_type_info();
protected:
__pointer_to_member_type_info(const __pointer_to_member_type_info&);
__pointer_to_member_type_info&
operator=(const __pointer_to_member_type_info&);
virtual bool
__pointer_catch(const __pbase_type_info* __thr_type, void** __thr_obj,
unsigned __outer) const;
};
class __base_class_type_info
{
public:
const __class_type_info* __base_type;
long __offset_flags;
enum __offset_flags_masks
{
__virtual_mask = 0x1,
__public_mask = 0x2,
__hwm_bit = 2,
__offset_shift = 8
};
bool
__is_virtual_p() const
{ return __offset_flags & __virtual_mask; }
bool
__is_public_p() const
{ return __offset_flags & __public_mask; }
ptrdiff_t
__offset() const
{
return static_cast<ptrdiff_t>(__offset_flags) >> __offset_shift;
}
};
class __class_type_info : public std::type_info
{
public:
explicit
__class_type_info (const char *__n) : type_info(__n) { }
virtual
~__class_type_info ();
enum __sub_kind
{
__unknown = 0,
__not_contained,
__contained_ambig,
__contained_virtual_mask = __base_class_type_info::__virtual_mask,
__contained_public_mask = __base_class_type_info::__public_mask,
__contained_mask = 1 << __base_class_type_info::__hwm_bit,
__contained_private = __contained_mask,
__contained_public = __contained_mask | __contained_public_mask
};
struct __upcast_result;
struct __dyncast_result;
protected:
virtual bool
__do_upcast(const __class_type_info* __dst_type, void**__obj_ptr) const;
virtual bool
__do_catch(const type_info* __thr_type, void** __thr_obj,
unsigned __outer) const;
public:
virtual bool
__do_upcast(const __class_type_info* __dst, const void* __obj,
__upcast_result& __restrict __result) const;
inline __sub_kind
__find_public_src(ptrdiff_t __src2dst, const void* __obj_ptr,
const __class_type_info* __src_type,
const void* __src_ptr) const;
virtual bool
__do_dyncast(ptrdiff_t __src2dst, __sub_kind __access_path,
const __class_type_info* __dst_type, const void* __obj_ptr,
const __class_type_info* __src_type, const void* __src_ptr,
__dyncast_result& __result) const;
virtual __sub_kind
__do_find_public_src(ptrdiff_t __src2dst, const void* __obj_ptr,
const __class_type_info* __src_type,
const void* __src_ptr) const;
};
class __si_class_type_info : public __class_type_info
{
public:
const __class_type_info* __base_type;
explicit
__si_class_type_info(const char *__n, const __class_type_info *__base)
: __class_type_info(__n), __base_type(__base) { }
virtual
~__si_class_type_info();
protected:
__si_class_type_info(const __si_class_type_info&);
__si_class_type_info&
operator=(const __si_class_type_info&);
virtual bool
__do_dyncast(ptrdiff_t __src2dst, __sub_kind __access_path,
const __class_type_info* __dst_type, const void* __obj_ptr,
const __class_type_info* __src_type, const void* __src_ptr,
__dyncast_result& __result) const;
virtual __sub_kind
__do_find_public_src(ptrdiff_t __src2dst, const void* __obj_ptr,
const __class_type_info* __src_type,
const void* __sub_ptr) const;
virtual bool
__do_upcast(const __class_type_info*__dst, const void*__obj,
__upcast_result& __restrict __result) const;
};
class __vmi_class_type_info : public __class_type_info
{
public:
unsigned int __flags;
unsigned int __base_count;
__base_class_type_info __base_info[1];
explicit
__vmi_class_type_info(const char* __n, int ___flags)
: __class_type_info(__n), __flags(___flags), __base_count(0) { }
virtual
~__vmi_class_type_info();
enum __flags_masks
{
__non_diamond_repeat_mask = 0x1,
__diamond_shaped_mask = 0x2,
__flags_unknown_mask = 0x10
};
protected:
virtual bool
__do_dyncast(ptrdiff_t __src2dst, __sub_kind __access_path,
const __class_type_info* __dst_type, const void* __obj_ptr,
const __class_type_info* __src_type, const void* __src_ptr,
__dyncast_result& __result) const;
virtual __sub_kind
__do_find_public_src(ptrdiff_t __src2dst, const void* __obj_ptr,
const __class_type_info* __src_type,
const void* __src_ptr) const;
virtual bool
__do_upcast(const __class_type_info* __dst, const void* __obj,
__upcast_result& __restrict __result) const;
};
struct __cxa_exception;
struct __cxa_refcounted_exception;
struct __cxa_dependent_exception;
struct __cxa_eh_globals;
extern "C"
{
void*
__dynamic_cast(const void* __src_ptr,
const __class_type_info* __src_type,
const __class_type_info* __dst_type,
ptrdiff_t __src2dst);
__cxa_eh_globals*
__cxa_get_globals() noexcept __attribute__ ((__const__));
__cxa_eh_globals*
__cxa_get_globals_fast() noexcept __attribute__ ((__const__));
void
__cxa_free_exception(void*) noexcept;
void
__cxa_throw(void*, std::type_info*, void ( *) (void *))
__attribute__((__noreturn__));
void*
__cxa_get_exception_ptr(void*) noexcept __attribute__ ((__pure__));
void*
__cxa_begin_catch(void*) noexcept;
void
__cxa_end_catch();
void
__cxa_rethrow() __attribute__((__noreturn__));
std::type_info*
__cxa_current_exception_type() noexcept __attribute__ ((__pure__));
__cxa_dependent_exception*
__cxa_allocate_dependent_exception() noexcept;
void
__cxa_free_dependent_exception(__cxa_dependent_exception*) noexcept;
}
class __foreign_exception
{
virtual ~__foreign_exception() throw();
virtual void __pure_dummy() = 0;
};
}
namespace abi = __cxxabiv1;
namespace __gnu_cxx
{
class recursive_init_error: public std::exception
{
public:
recursive_init_error() noexcept;
virtual ~recursive_init_error() noexcept;
};
}
#pragma GCC visibility pop
namespace testing {
namespace internal {
inline std::string CanonicalizeForStdLibVersioning(std::string s) {
static const char prefix[] = "std::__";
if (s.compare(0, strlen(prefix), prefix) == 0) {
std::string::size_type end = s.find("::", strlen(prefix));
if (end != s.npos) {
s.erase(strlen("std"), end - strlen("std"));
}
}
return s;
}
inline std::string GetTypeName(const std::type_info& type) {
const char* const name = type.name();
int status = 0;
using abi::__cxa_demangle;
char* const readable_name = __cxa_demangle(name, nullptr, nullptr, &status);
const std::string name_str(status == 0 ? readable_name : name);
free(readable_name);
return CanonicalizeForStdLibVersioning(name_str);
}
template <typename T>
std::string GetTypeName() {
return GetTypeName(typeid(T));
}
struct None {};
template <template <typename T> class Tmpl>
struct TemplateSel {
template <typename T>
struct Bind {
typedef Tmpl<T> type;
};
};
template <template <typename T> class Head_, template <typename T> class... Tail_>
struct Templates {
using Head = TemplateSel<Head_>;
using Tail = Templates<Tail_...>;
};
template <template <typename T> class Head_>
struct Templates<Head_> {
using Head = TemplateSel<Head_>;
using Tail = None;
};
template <typename Head_, typename... Tail_>
struct Types {
using Head = Head_;
using Tail = Types<Tail_...>;
};
template <typename Head_>
struct Types<Head_> {
using Head = Head_;
using Tail = None;
};
template <typename... Ts>
struct ProxyTypeList {
using type = Types<Ts...>;
};
template <typename>
struct is_proxy_type_list : std::false_type {};
template <typename... Ts>
struct is_proxy_type_list<ProxyTypeList<Ts...>> : std::true_type {};
template <typename T>
struct GenerateTypeList {
private:
using proxy = typename std::conditional<is_proxy_type_list<T>::value, T,
ProxyTypeList<T>>::type;
public:
using type = typename proxy::type;
};
}
template <typename... Ts>
using Types = internal::ProxyTypeList<Ts...>;
}
namespace proto2 {
class MessageLite;
}
namespace testing {
class AssertionResult;
class Message;
class Test;
class TestInfo;
class TestPartResult;
class UnitTest;
template <typename T>
::std::string PrintToString(const T& value);
namespace internal {
struct TraceInfo;
class TestInfoImpl;
class UnitTestImpl;
__attribute__((visibility("default"))) extern const char kStackTraceMarker[];
class IgnoredValue {
struct Sink {};
public:
template <typename T,
typename std::enable_if<!std::is_convertible<T, Sink>::value,
int>::type = 0>
IgnoredValue(const T& ) {}
};
__attribute__((visibility("default"))) std::string AppendUserMessage(const std::string& gtest_msg,
const Message& user_msg);
class __attribute__((visibility("default"))) GoogleTestFailureException : public ::std::runtime_error {
public:
explicit GoogleTestFailureException(const TestPartResult& failure);
};
namespace edit_distance {
enum EditType { kMatch, kAdd, kRemove, kReplace };
__attribute__((visibility("default"))) std::vector<EditType> CalculateOptimalEdits(
const std::vector<size_t>& left, const std::vector<size_t>& right);
__attribute__((visibility("default"))) std::vector<EditType> CalculateOptimalEdits(
const std::vector<std::string>& left,
const std::vector<std::string>& right);
__attribute__((visibility("default"))) std::string CreateUnifiedDiff(const std::vector<std::string>& left,
const std::vector<std::string>& right,
size_t context = 2);
}
__attribute__((visibility("default"))) AssertionResult EqFailure(const char* expected_expression,
const char* actual_expression,
const std::string& expected_value,
const std::string& actual_value,
bool ignoring_case);
__attribute__((visibility("default"))) std::string GetBoolAssertionFailureMessage(
const AssertionResult& assertion_result, const char* expression_text,
const char* actual_predicate_value, const char* expected_predicate_value);
template <typename RawType>
class FloatingPoint {
public:
typedef typename TypeWithSize<sizeof(RawType)>::UInt Bits;
static const size_t kBitCount = 8 * sizeof(RawType);
static const size_t kFractionBitCount =
std::numeric_limits<RawType>::digits - 1;
static const size_t kExponentBitCount = kBitCount - 1 - kFractionBitCount;
static const Bits kSignBitMask = static_cast<Bits>(1) << (kBitCount - 1);
static const Bits kFractionBitMask = ~static_cast<Bits>(0) >>
(kExponentBitCount + 1);
static const Bits kExponentBitMask = ~(kSignBitMask | kFractionBitMask);
static const uint32_t kMaxUlps = 4;
explicit FloatingPoint(const RawType& x) { u_.value_ = x; }
static RawType ReinterpretBits(const Bits bits) {
FloatingPoint fp(0);
fp.u_.bits_ = bits;
return fp.u_.value_;
}
static RawType Infinity() { return ReinterpretBits(kExponentBitMask); }
static RawType Max();
const Bits& bits() const { return u_.bits_; }
Bits exponent_bits() const { return kExponentBitMask & u_.bits_; }
Bits fraction_bits() const { return kFractionBitMask & u_.bits_; }
Bits sign_bit() const { return kSignBitMask & u_.bits_; }
bool is_nan() const {
return (exponent_bits() == kExponentBitMask) && (fraction_bits() != 0);
}
bool AlmostEquals(const FloatingPoint& rhs) const {
if (is_nan() || rhs.is_nan()) return false;
return DistanceBetweenSignAndMagnitudeNumbers(u_.bits_, rhs.u_.bits_) <=
kMaxUlps;
}
private:
union FloatingPointUnion {
RawType value_;
Bits bits_;
};
static Bits SignAndMagnitudeToBiased(const Bits& sam) {
if (kSignBitMask & sam) {
return ~sam + 1;
} else {
return kSignBitMask | sam;
}
}
static Bits DistanceBetweenSignAndMagnitudeNumbers(const Bits& sam1,
const Bits& sam2) {
const Bits biased1 = SignAndMagnitudeToBiased(sam1);
const Bits biased2 = SignAndMagnitudeToBiased(sam2);
return (biased1 >= biased2) ? (biased1 - biased2) : (biased2 - biased1);
}
FloatingPointUnion u_;
};
template <>
inline float FloatingPoint<float>::Max() {
return 3.40282346638528859811704183484516925e+38F;
}
template <>
inline double FloatingPoint<double>::Max() {
return double(1.79769313486231570814527423731704357e+308L);
}
typedef FloatingPoint<float> Float;
typedef FloatingPoint<double> Double;
typedef const void* TypeId;
template <typename T>
class TypeIdHelper {
public:
static bool dummy_;
};
template <typename T>
bool TypeIdHelper<T>::dummy_ = false;
template <typename T>
TypeId GetTypeId() {
return &(TypeIdHelper<T>::dummy_);
}
__attribute__((visibility("default"))) TypeId GetTestTypeId();
class TestFactoryBase {
public:
virtual ~TestFactoryBase() {}
virtual Test* CreateTest() = 0;
protected:
TestFactoryBase() {}
private:
TestFactoryBase(const TestFactoryBase&) = delete;
TestFactoryBase& operator=(const TestFactoryBase&) = delete;
};
template <class TestClass>
class TestFactoryImpl : public TestFactoryBase {
public:
Test* CreateTest() override { return new TestClass; }
};
using SetUpTestSuiteFunc = void (*)();
using TearDownTestSuiteFunc = void (*)();
struct CodeLocation {
CodeLocation(const std::string& a_file, int a_line)
: file(a_file), line(a_line) {}
std::string file;
int line;
};
using SetUpTearDownSuiteFuncType = void (*)();
inline SetUpTearDownSuiteFuncType GetNotDefaultOrNull(
SetUpTearDownSuiteFuncType a, SetUpTearDownSuiteFuncType def) {
return a == def ? nullptr : a;
}
template <typename T>
struct SuiteApiResolver : T {
using Test =
typename std::conditional<sizeof(T) != 0, ::testing::Test, void>::type;
static SetUpTearDownSuiteFuncType GetSetUpCaseOrSuite(const char* filename,
int line_num) {
SetUpTearDownSuiteFuncType test_case_fp =
GetNotDefaultOrNull(&T::SetUpTestCase, &Test::SetUpTestCase);
SetUpTearDownSuiteFuncType test_suite_fp =
GetNotDefaultOrNull(&T::SetUpTestSuite, &Test::SetUpTestSuite);
switch (0) case 0: default: if (::testing::internal::IsTrue(!test_case_fp || !test_suite_fp)) ; else ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-internal.h", 528) .GetStream() << "Condition " "!test_case_fp || !test_suite_fp" " failed. "
<< "Test can not provide both SetUpTestSuite and SetUpTestCase, please "
"make sure there is only one present at "
<< filename << ":" << line_num;
return test_case_fp != nullptr ? test_case_fp : test_suite_fp;
}
static SetUpTearDownSuiteFuncType GetTearDownCaseOrSuite(const char* filename,
int line_num) {
SetUpTearDownSuiteFuncType test_case_fp =
GetNotDefaultOrNull(&T::TearDownTestCase, &Test::TearDownTestCase);
SetUpTearDownSuiteFuncType test_suite_fp =
GetNotDefaultOrNull(&T::TearDownTestSuite, &Test::TearDownTestSuite);
switch (0) case 0: default: if (::testing::internal::IsTrue(!test_case_fp || !test_suite_fp)) ; else ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-internal.h", 549) .GetStream() << "Condition " "!test_case_fp || !test_suite_fp" " failed. "
<< "Test can not provide both TearDownTestSuite and TearDownTestCase,"
" please make sure there is only one present at"
<< filename << ":" << line_num;
return test_case_fp != nullptr ? test_case_fp : test_suite_fp;
}
};
__attribute__((visibility("default"))) TestInfo* MakeAndRegisterTestInfo(
const char* test_suite_name, const char* name, const char* type_param,
const char* value_param, CodeLocation code_location,
TypeId fixture_class_id, SetUpTestSuiteFunc set_up_tc,
TearDownTestSuiteFunc tear_down_tc, TestFactoryBase* factory);
__attribute__((visibility("default"))) bool SkipPrefix(const char* prefix, const char** pstr);
class __attribute__((visibility("default"))) TypedTestSuitePState {
public:
TypedTestSuitePState() : registered_(false) {}
bool AddTestName(const char* file, int line, const char* case_name,
const char* test_name) {
if (registered_) {
fprintf(stderr,
"%s Test %s must be defined before "
"REGISTER_TYPED_TEST_SUITE_P(%s, ...).\n",
FormatFileLocation(file, line).c_str(), test_name, case_name);
fflush(stderr);
posix::Abort();
}
registered_tests_.insert(
::std::make_pair(test_name, CodeLocation(file, line)));
return true;
}
bool TestExists(const std::string& test_name) const {
return registered_tests_.count(test_name) > 0;
}
const CodeLocation& GetCodeLocation(const std::string& test_name) const {
RegisteredTestsMap::const_iterator it = registered_tests_.find(test_name);
switch (0) case 0: default: if (::testing::internal::IsTrue(it != registered_tests_.end())) ; else ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-internal.h", 624) .GetStream() << "Condition " "it != registered_tests_.end()" " failed. ";
return it->second;
}
const char* VerifyRegisteredTestNames(const char* test_suite_name,
const char* file, int line,
const char* registered_tests);
private:
typedef ::std::map<std::string, CodeLocation, std::less<>> RegisteredTestsMap;
bool registered_;
RegisteredTestsMap registered_tests_;
};
using TypedTestCasePState = TypedTestSuitePState;
inline const char* SkipComma(const char* str) {
const char* comma = strchr(str, ',');
if (comma == nullptr) {
return nullptr;
}
while (IsSpace(*(++comma))) {
}
return comma;
}
inline std::string GetPrefixUntilComma(const char* str) {
const char* comma = strchr(str, ',');
return comma == nullptr ? str : std::string(str, comma);
}
void SplitString(const ::std::string& str, char delimiter,
::std::vector<::std::string>* dest);
struct DefaultNameGenerator {
template <typename T>
static std::string GetName(int i) {
return StreamableToString(i);
}
};
template <typename Provided = DefaultNameGenerator>
struct NameGeneratorSelector {
typedef Provided type;
};
template <typename NameGenerator>
void GenerateNamesRecursively(internal::None, std::vector<std::string>*, int) {}
template <typename NameGenerator, typename Types>
void GenerateNamesRecursively(Types, std::vector<std::string>* result, int i) {
result->push_back(NameGenerator::template GetName<typename Types::Head>(i));
GenerateNamesRecursively<NameGenerator>(typename Types::Tail(), result,
i + 1);
}
template <typename NameGenerator, typename Types>
std::vector<std::string> GenerateNames() {
std::vector<std::string> result;
GenerateNamesRecursively<NameGenerator>(Types(), &result, 0);
return result;
}
template <template <typename T> class Fixture, class TestSel, typename Types>
class TypeParameterizedTest {
public:
static bool Register(const char* prefix, const CodeLocation& code_location,
const char* case_name, const char* test_names, int index,
const std::vector<std::string>& type_names =
GenerateNames<DefaultNameGenerator, Types>()) {
typedef typename Types::Head Type;
typedef Fixture<Type> FixtureClass;
typedef typename TestSel::template Bind<Type>::type TestClass;
MakeAndRegisterTestInfo(
(std::string(prefix) + (prefix[0] == '\0' ? "" : "/") + case_name +
"/" + type_names[static_cast<size_t>(index)])
.c_str(),
StripTrailingSpaces(GetPrefixUntilComma(test_names)).c_str(),
GetTypeName<Type>().c_str(),
nullptr,
code_location, GetTypeId<FixtureClass>(),
SuiteApiResolver<TestClass>::GetSetUpCaseOrSuite(
code_location.file.c_str(), code_location.line),
SuiteApiResolver<TestClass>::GetTearDownCaseOrSuite(
code_location.file.c_str(), code_location.line),
new TestFactoryImpl<TestClass>);
return TypeParameterizedTest<Fixture, TestSel,
typename Types::Tail>::Register(prefix,
code_location,
case_name,
test_names,
index + 1,
type_names);
}
};
template <template <typename T> class Fixture, class TestSel>
class TypeParameterizedTest<Fixture, TestSel, internal::None> {
public:
static bool Register(const char* , const CodeLocation&,
const char* , const char* ,
int ,
const std::vector<std::string>& =
std::vector<std::string>() ) {
return true;
}
};
__attribute__((visibility("default"))) void RegisterTypeParameterizedTestSuite(const char* test_suite_name,
CodeLocation code_location);
__attribute__((visibility("default"))) void RegisterTypeParameterizedTestSuiteInstantiation(
const char* case_name);
template <template <typename T> class Fixture, typename Tests, typename Types>
class TypeParameterizedTestSuite {
public:
static bool Register(const char* prefix, CodeLocation code_location,
const TypedTestSuitePState* state, const char* case_name,
const char* test_names,
const std::vector<std::string>& type_names =
GenerateNames<DefaultNameGenerator, Types>()) {
RegisterTypeParameterizedTestSuiteInstantiation(case_name);
std::string test_name =
StripTrailingSpaces(GetPrefixUntilComma(test_names));
if (!state->TestExists(test_name)) {
fprintf(stderr, "Failed to get code location for test %s.%s at %s.",
case_name, test_name.c_str(),
FormatFileLocation(code_location.file.c_str(), code_location.line)
.c_str());
fflush(stderr);
posix::Abort();
}
const CodeLocation& test_location = state->GetCodeLocation(test_name);
typedef typename Tests::Head Head;
TypeParameterizedTest<Fixture, Head, Types>::Register(
prefix, test_location, case_name, test_names, 0, type_names);
return TypeParameterizedTestSuite<Fixture, typename Tests::Tail,
Types>::Register(prefix, code_location,
state, case_name,
SkipComma(test_names),
type_names);
}
};
template <template <typename T> class Fixture, typename Types>
class TypeParameterizedTestSuite<Fixture, internal::None, Types> {
public:
static bool Register(const char* , const CodeLocation&,
const TypedTestSuitePState* ,
const char* , const char* ,
const std::vector<std::string>& =
std::vector<std::string>() ) {
return true;
}
};
__attribute__((visibility("default"))) std::string GetCurrentOsStackTraceExceptTop(int skip_count);
__attribute__((visibility("default"))) bool AlwaysTrue();
inline bool AlwaysFalse() { return !AlwaysTrue(); }
struct __attribute__((visibility("default"))) ConstCharPtr {
ConstCharPtr(const char* str) : value(str) {}
operator bool() const { return true; }
const char* value;
};
struct TrueWithString {
TrueWithString() = default;
explicit TrueWithString(const char* str) : value(str) {}
explicit TrueWithString(const std::string& str) : value(str) {}
explicit operator bool() const { return true; }
std::string value;
};
class __attribute__((visibility("default"))) Random {
public:
static const uint32_t kMaxRange = 1u << 31;
explicit Random(uint32_t seed) : state_(seed) {}
void Reseed(uint32_t seed) { state_ = seed; }
uint32_t Generate(uint32_t range);
private:
uint32_t state_;
Random(const Random&) = delete;
Random& operator=(const Random&) = delete;
};
template <typename T>
class HasDebugStringAndShortDebugString {
private:
template <typename C>
static auto CheckDebugString(C*) -> typename std::is_same<
std::string, decltype(std::declval<const C>().DebugString())>::type;
template <typename>
static std::false_type CheckDebugString(...);
template <typename C>
static auto CheckShortDebugString(C*) -> typename std::is_same<
std::string, decltype(std::declval<const C>().ShortDebugString())>::type;
template <typename>
static std::false_type CheckShortDebugString(...);
using HasDebugStringType = decltype(CheckDebugString<T>(nullptr));
using HasShortDebugStringType = decltype(CheckShortDebugString<T>(nullptr));
public:
static constexpr bool value =
HasDebugStringType::value && HasShortDebugStringType::value;
};
template <typename T>
constexpr bool HasDebugStringAndShortDebugString<T>::value;
typedef int IsContainer;
template <class C,
class Iterator = decltype(::std::declval<const C&>().begin()),
class = decltype(::std::declval<const C&>().end()),
class = decltype(++::std::declval<Iterator&>()),
class = decltype(*::std::declval<Iterator>()),
class = typename C::const_iterator>
IsContainer IsContainerTest(int ) {
return 0;
}
typedef char IsNotContainer;
template <class C>
IsNotContainer IsContainerTest(long ) {
return '\0';
}
template <typename T>
struct IsHashTable {
private:
template <typename U>
static char test(typename U::hasher*, typename U::reverse_iterator*);
template <typename U>
static int test(typename U::hasher*, ...);
template <typename U>
static char test(...);
public:
static const bool value = sizeof(test<T>(nullptr, nullptr)) == sizeof(int);
};
template <typename T>
const bool IsHashTable<T>::value;
template <typename C,
bool = sizeof(IsContainerTest<C>(0)) == sizeof(IsContainer)>
struct IsRecursiveContainerImpl;
template <typename C>
struct IsRecursiveContainerImpl<C, false> : public std::false_type {};
template <typename C>
struct IsRecursiveContainerImpl<C, true> {
using value_type = decltype(*std::declval<typename C::const_iterator>());
using type =
std::is_same<typename std::remove_const<
typename std::remove_reference<value_type>::type>::type,
C>;
};
template <typename C>
struct IsRecursiveContainer : public IsRecursiveContainerImpl<C>::type {};
template <typename T, typename U>
bool ArrayEq(const T* lhs, size_t size, const U* rhs);
template <typename T, typename U>
inline bool ArrayEq(const T& lhs, const U& rhs) {
return lhs == rhs;
}
template <typename T, typename U, size_t N>
inline bool ArrayEq(const T (&lhs)[N], const U (&rhs)[N]) {
return internal::ArrayEq(lhs, N, rhs);
}
template <typename T, typename U>
bool ArrayEq(const T* lhs, size_t size, const U* rhs) {
for (size_t i = 0; i != size; i++) {
if (!internal::ArrayEq(lhs[i], rhs[i])) return false;
}
return true;
}
template <typename Iter, typename Element>
Iter ArrayAwareFind(Iter begin, Iter end, const Element& elem) {
for (Iter it = begin; it != end; ++it) {
if (internal::ArrayEq(*it, elem)) return it;
}
return end;
}
template <typename T, typename U>
void CopyArray(const T* from, size_t size, U* to);
template <typename T, typename U>
inline void CopyArray(const T& from, U* to) {
*to = from;
}
template <typename T, typename U, size_t N>
inline void CopyArray(const T (&from)[N], U (*to)[N]) {
internal::CopyArray(from, N, *to);
}
template <typename T, typename U>
void CopyArray(const T* from, size_t size, U* to) {
for (size_t i = 0; i != size; i++) {
internal::CopyArray(from[i], to + i);
}
}
struct RelationToSourceReference {};
struct RelationToSourceCopy {};
template <typename Element>
class NativeArray {
public:
typedef Element value_type;
typedef Element* iterator;
typedef const Element* const_iterator;
NativeArray(const Element* array, size_t count, RelationToSourceReference) {
InitRef(array, count);
}
NativeArray(const Element* array, size_t count, RelationToSourceCopy) {
InitCopy(array, count);
}
NativeArray(const NativeArray& rhs) {
(this->*rhs.clone_)(rhs.array_, rhs.size_);
}
~NativeArray() {
if (clone_ != &NativeArray::InitRef) delete[] array_;
}
size_t size() const { return size_; }
const_iterator begin() const { return array_; }
const_iterator end() const { return array_ + size_; }
bool operator==(const NativeArray& rhs) const {
return size() == rhs.size() && ArrayEq(begin(), size(), rhs.begin());
}
private:
static_assert(!std::is_const<Element>::value, "Type must not be const");
static_assert(!std::is_reference<Element>::value,
"Type must not be a reference");
void InitCopy(const Element* array, size_t a_size) {
Element* const copy = new Element[a_size];
CopyArray(array, a_size, copy);
array_ = copy;
size_ = a_size;
clone_ = &NativeArray::InitCopy;
}
void InitRef(const Element* array, size_t a_size) {
array_ = array;
size_ = a_size;
clone_ = &NativeArray::InitRef;
}
const Element* array_;
size_t size_;
void (NativeArray::*clone_)(const Element*, size_t);
};
template <size_t... Is>
struct IndexSequence {
using type = IndexSequence;
};
template <bool plus_one, typename T, size_t sizeofT>
struct DoubleSequence;
template <size_t... I, size_t sizeofT>
struct DoubleSequence<true, IndexSequence<I...>, sizeofT> {
using type = IndexSequence<I..., (sizeofT + I)..., 2 * sizeofT>;
};
template <size_t... I, size_t sizeofT>
struct DoubleSequence<false, IndexSequence<I...>, sizeofT> {
using type = IndexSequence<I..., (sizeofT + I)...>;
};
template <size_t N>
struct MakeIndexSequenceImpl
: DoubleSequence<N % 2 == 1, typename MakeIndexSequenceImpl<N / 2>::type,
N / 2>::type {};
template <>
struct MakeIndexSequenceImpl<0> : IndexSequence<> {};
template <size_t N>
using MakeIndexSequence = typename MakeIndexSequenceImpl<N>::type;
template <typename... T>
using IndexSequenceFor = typename MakeIndexSequence<sizeof...(T)>::type;
template <size_t>
struct Ignore {
Ignore(...);
};
template <typename>
struct ElemFromListImpl;
template <size_t... I>
struct ElemFromListImpl<IndexSequence<I...>> {
template <typename R>
static R Apply(Ignore<0 * I>..., R (*)(), ...);
};
template <size_t N, typename... T>
struct ElemFromList {
using type =
decltype(ElemFromListImpl<typename MakeIndexSequence<N>::type>::Apply(
static_cast<T (*)()>(nullptr)...));
};
struct FlatTupleConstructTag {};
template <typename... T>
class FlatTuple;
template <typename Derived, size_t I>
struct FlatTupleElemBase;
template <typename... T, size_t I>
struct FlatTupleElemBase<FlatTuple<T...>, I> {
using value_type = typename ElemFromList<I, T...>::type;
FlatTupleElemBase() = default;
template <typename Arg>
explicit FlatTupleElemBase(FlatTupleConstructTag, Arg&& t)
: value(std::forward<Arg>(t)) {}
value_type value;
};
template <typename Derived, typename Idx>
struct FlatTupleBase;
template <size_t... Idx, typename... T>
struct FlatTupleBase<FlatTuple<T...>, IndexSequence<Idx...>>
: FlatTupleElemBase<FlatTuple<T...>, Idx>... {
using Indices = IndexSequence<Idx...>;
FlatTupleBase() = default;
template <typename... Args>
explicit FlatTupleBase(FlatTupleConstructTag, Args&&... args)
: FlatTupleElemBase<FlatTuple<T...>, Idx>(FlatTupleConstructTag{},
std::forward<Args>(args))... {}
template <size_t I>
const typename ElemFromList<I, T...>::type& Get() const {
return FlatTupleElemBase<FlatTuple<T...>, I>::value;
}
template <size_t I>
typename ElemFromList<I, T...>::type& Get() {
return FlatTupleElemBase<FlatTuple<T...>, I>::value;
}
template <typename F>
auto Apply(F&& f) -> decltype(std::forward<F>(f)(this->Get<Idx>()...)) {
return std::forward<F>(f)(Get<Idx>()...);
}
template <typename F>
auto Apply(F&& f) const -> decltype(std::forward<F>(f)(this->Get<Idx>()...)) {
return std::forward<F>(f)(Get<Idx>()...);
}
};
template <typename... T>
class FlatTuple
: private FlatTupleBase<FlatTuple<T...>,
typename MakeIndexSequence<sizeof...(T)>::type> {
using Indices = typename FlatTupleBase<
FlatTuple<T...>, typename MakeIndexSequence<sizeof...(T)>::type>::Indices;
public:
FlatTuple() = default;
template <typename... Args>
explicit FlatTuple(FlatTupleConstructTag tag, Args&&... args)
: FlatTuple::FlatTupleBase(tag, std::forward<Args>(args)...) {}
using FlatTuple::FlatTupleBase::Apply;
using FlatTuple::FlatTupleBase::Get;
};
__attribute__((deprecated("INSTANTIATE_TEST_CASE_P is deprecated, please use " "INSTANTIATE_TEST_SUITE_P")))
constexpr bool InstantiateTestCase_P_IsDeprecated() { return true; }
__attribute__((deprecated("TYPED_TEST_CASE_P is deprecated, please use " "TYPED_TEST_SUITE_P")))
constexpr bool TypedTestCase_P_IsDeprecated() { return true; }
__attribute__((deprecated("TYPED_TEST_CASE is deprecated, please use " "TYPED_TEST_SUITE")))
constexpr bool TypedTestCaseIsDeprecated() { return true; }
__attribute__((deprecated("REGISTER_TYPED_TEST_CASE_P is deprecated, please use " "REGISTER_TYPED_TEST_SUITE_P")))
constexpr bool RegisterTypedTestCase_P_IsDeprecated() { return true; }
__attribute__((deprecated("INSTANTIATE_TYPED_TEST_CASE_P is deprecated, please use " "INSTANTIATE_TYPED_TEST_SUITE_P")))
constexpr bool InstantiateTypedTestCase_P_IsDeprecated() { return true; }
}
}
namespace std {
template <typename... Ts>
struct tuple_size<testing::internal::FlatTuple<Ts...>>
: std::integral_constant<size_t, sizeof...(Ts)> {};
}
namespace testing {
namespace internal {
class NeverThrown {
public:
const char* what() const noexcept {
return "this exception should never be thrown";
}
};
}
}
namespace testing {
namespace internal {
template <typename T>
void UniversalPrint(const T& value, ::std::ostream* os);
struct ContainerPrinter {
template <typename T,
typename = typename std::enable_if<
(sizeof(IsContainerTest<T>(0)) == sizeof(IsContainer)) &&
!IsRecursiveContainer<T>::value>::type>
static void PrintValue(const T& container, std::ostream* os) {
const size_t kMaxCount = 32;
*os << '{';
size_t count = 0;
for (auto&& elem : container) {
if (count > 0) {
*os << ',';
if (count == kMaxCount) {
*os << " ...";
break;
}
}
*os << ' ';
internal::UniversalPrint(elem, os);
++count;
}
if (count > 0) {
*os << ' ';
}
*os << '}';
}
};
struct FunctionPointerPrinter {
template <typename T, typename = typename std::enable_if<
std::is_function<T>::value>::type>
static void PrintValue(T* p, ::std::ostream* os) {
if (p == nullptr) {
*os << "NULL";
} else {
*os << reinterpret_cast<const void*>(p);
}
}
};
struct PointerPrinter {
template <typename T>
static void PrintValue(T* p, ::std::ostream* os) {
if (p == nullptr) {
*os << "NULL";
} else {
*os << p;
}
}
};
namespace internal_stream_operator_without_lexical_name_lookup {
struct LookupBlocker {};
void operator<<(LookupBlocker, LookupBlocker);
struct StreamPrinter {
template <typename T,
typename = typename std::enable_if<
!std::is_member_pointer<T>::value>::type,
typename = decltype(std::declval<std::ostream&>()
<< std::declval<const T&>())>
static void PrintValue(const T& value, ::std::ostream* os) {
*os << value;
}
};
}
struct ProtobufPrinter {
static const size_t kProtobufOneLinerMaxLength = 50;
template <typename T,
typename = typename std::enable_if<
internal::HasDebugStringAndShortDebugString<T>::value>::type>
static void PrintValue(const T& value, ::std::ostream* os) {
std::string pretty_str = value.ShortDebugString();
if (pretty_str.length() > kProtobufOneLinerMaxLength) {
pretty_str = "\n" + value.DebugString();
}
*os << ("<" + pretty_str + ">");
}
};
struct ConvertibleToIntegerPrinter {
static void PrintValue(internal::BiggestInt value, ::std::ostream* os) {
*os << value;
}
};
struct ConvertibleToStringViewPrinter {
static void PrintValue(internal::StringView value, ::std::ostream* os) {
internal::UniversalPrint(value, os);
}
};
__attribute__((visibility("default"))) void PrintBytesInObjectTo(const unsigned char* obj_bytes,
size_t count, ::std::ostream* os);
struct RawBytesPrinter {
template <typename T, size_t = sizeof(T)>
static void PrintValue(const T& value, ::std::ostream* os) {
PrintBytesInObjectTo(
static_cast<const unsigned char*>(
reinterpret_cast<const void*>(std::addressof(value))),
sizeof(value), os);
}
};
struct FallbackPrinter {
template <typename T>
static void PrintValue(const T&, ::std::ostream* os) {
*os << "(incomplete type)";
}
};
template <typename T, typename E, typename Printer, typename... Printers>
struct FindFirstPrinter : FindFirstPrinter<T, E, Printers...> {};
template <typename T, typename Printer, typename... Printers>
struct FindFirstPrinter<
T, decltype(Printer::PrintValue(std::declval<const T&>(), nullptr)),
Printer, Printers...> {
using type = Printer;
};
template <typename T>
void PrintWithFallback(const T& value, ::std::ostream* os) {
using Printer = typename FindFirstPrinter<
T, void, ContainerPrinter, FunctionPointerPrinter, PointerPrinter,
internal_stream_operator_without_lexical_name_lookup::StreamPrinter,
ProtobufPrinter, ConvertibleToIntegerPrinter,
ConvertibleToStringViewPrinter, RawBytesPrinter, FallbackPrinter>::type;
Printer::PrintValue(value, os);
}
template <typename ToPrint, typename OtherOperand>
class FormatForComparison {
public:
static ::std::string Format(const ToPrint& value) {
return ::testing::PrintToString(value);
}
};
template <typename ToPrint, size_t N, typename OtherOperand>
class FormatForComparison<ToPrint[N], OtherOperand> {
public:
static ::std::string Format(const ToPrint* value) {
return FormatForComparison<const ToPrint*, OtherOperand>::Format(value);
}
};
template <typename OtherOperand> class FormatForComparison<char*, OtherOperand> { public: static ::std::string Format(char* value) { return ::testing::PrintToString(static_cast<const void*>(value)); } };
template <typename OtherOperand> class FormatForComparison<const char*, OtherOperand> { public: static ::std::string Format(const char* value) { return ::testing::PrintToString(static_cast<const void*>(value)); } };
template <typename OtherOperand> class FormatForComparison<wchar_t*, OtherOperand> { public: static ::std::string Format(wchar_t* value) { return ::testing::PrintToString(static_cast<const void*>(value)); } };
template <typename OtherOperand> class FormatForComparison<const wchar_t*, OtherOperand> { public: static ::std::string Format(const wchar_t* value) { return ::testing::PrintToString(static_cast<const void*>(value)); } };
template <typename OtherOperand> class FormatForComparison<char16_t*, OtherOperand> { public: static ::std::string Format(char16_t* value) { return ::testing::PrintToString(static_cast<const void*>(value)); } };
template <typename OtherOperand> class FormatForComparison<const char16_t*, OtherOperand> { public: static ::std::string Format(const char16_t* value) { return ::testing::PrintToString(static_cast<const void*>(value)); } };
template <typename OtherOperand> class FormatForComparison<char32_t*, OtherOperand> { public: static ::std::string Format(char32_t* value) { return ::testing::PrintToString(static_cast<const void*>(value)); } };
template <typename OtherOperand> class FormatForComparison<const char32_t*, OtherOperand> { public: static ::std::string Format(const char32_t* value) { return ::testing::PrintToString(static_cast<const void*>(value)); } };
template <> class FormatForComparison<char*, ::std::string> { public: static ::std::string Format(char* value) { return ::testing::PrintToString(value); } };
template <> class FormatForComparison<const char*, ::std::string> { public: static ::std::string Format(const char* value) { return ::testing::PrintToString(value); } };
template <> class FormatForComparison<char16_t*, ::std::u16string> { public: static ::std::string Format(char16_t* value) { return ::testing::PrintToString(value); } };
template <> class FormatForComparison<const char16_t*, ::std::u16string> { public: static ::std::string Format(const char16_t* value) { return ::testing::PrintToString(value); } };
template <> class FormatForComparison<char32_t*, ::std::u32string> { public: static ::std::string Format(char32_t* value) { return ::testing::PrintToString(value); } };
template <> class FormatForComparison<const char32_t*, ::std::u32string> { public: static ::std::string Format(const char32_t* value) { return ::testing::PrintToString(value); } };
template <> class FormatForComparison<wchar_t*, ::std::wstring> { public: static ::std::string Format(wchar_t* value) { return ::testing::PrintToString(value); } };
template <> class FormatForComparison<const wchar_t*, ::std::wstring> { public: static ::std::string Format(const wchar_t* value) { return ::testing::PrintToString(value); } };
template <typename T1, typename T2>
std::string FormatForComparisonFailureMessage(const T1& value,
const T2& ) {
return FormatForComparison<T1, T2>::Format(value);
}
template <typename T>
class UniversalPrinter;
template <typename T>
void PrintTo(const T& value, ::std::ostream* os) {
internal::PrintWithFallback(value, os);
}
__attribute__((visibility("default"))) void PrintTo(unsigned char c, ::std::ostream* os);
__attribute__((visibility("default"))) void PrintTo(signed char c, ::std::ostream* os);
inline void PrintTo(char c, ::std::ostream* os) {
PrintTo(static_cast<unsigned char>(c), os);
}
inline void PrintTo(bool x, ::std::ostream* os) {
*os << (x ? "true" : "false");
}
__attribute__((visibility("default"))) void PrintTo(wchar_t wc, ::std::ostream* os);
__attribute__((visibility("default"))) void PrintTo(char32_t c, ::std::ostream* os);
inline void PrintTo(char16_t c, ::std::ostream* os) {
PrintTo(ImplicitCast_<char32_t>(c), os);
}
template <typename FloatType>
int AppropriateResolution(FloatType val) {
int full = std::numeric_limits<FloatType>::max_digits10;
if (val < 0) val = -val;
if (val < 1000000) {
FloatType mulfor6 = 1e10;
if (val >= 100000.0) {
mulfor6 = 1.0;
} else if (val >= 10000.0) {
mulfor6 = 1e1;
} else if (val >= 1000.0) {
mulfor6 = 1e2;
} else if (val >= 100.0) {
mulfor6 = 1e3;
} else if (val >= 10.0) {
mulfor6 = 1e4;
} else if (val >= 1.0) {
mulfor6 = 1e5;
} else if (val >= 0.1) {
mulfor6 = 1e6;
} else if (val >= 0.01) {
mulfor6 = 1e7;
} else if (val >= 0.001) {
mulfor6 = 1e8;
} else if (val >= 0.0001) {
mulfor6 = 1e9;
}
if (static_cast<FloatType>(static_cast<int32_t>(val * mulfor6 + 0.5)) / mulfor6 == val) return 6;
} else if (val < 1e10) {
FloatType divfor6 = 1.0;
if (val >= 1e9) {
divfor6 = 10000;
} else if (val >= 1e8) {
divfor6 = 1000;
} else if (val >= 1e7) {
divfor6 = 100;
} else if (val >= 1e6) {
divfor6 = 10;
}
if (static_cast<FloatType>(static_cast<int32_t>(val / divfor6 + 0.5)) * divfor6 == val) return 6;
}
return full;
}
inline void PrintTo(float f, ::std::ostream* os) {
auto old_precision = os->precision();
os->precision(AppropriateResolution(f));
*os << f;
os->precision(old_precision);
}
inline void PrintTo(double d, ::std::ostream* os) {
auto old_precision = os->precision();
os->precision(AppropriateResolution(d));
*os << d;
os->precision(old_precision);
}
__attribute__((visibility("default"))) void PrintTo(const char* s, ::std::ostream* os);
inline void PrintTo(char* s, ::std::ostream* os) {
PrintTo(ImplicitCast_<const char*>(s), os);
}
inline void PrintTo(const signed char* s, ::std::ostream* os) {
PrintTo(ImplicitCast_<const void*>(s), os);
}
inline void PrintTo(signed char* s, ::std::ostream* os) {
PrintTo(ImplicitCast_<const void*>(s), os);
}
inline void PrintTo(const unsigned char* s, ::std::ostream* os) {
PrintTo(ImplicitCast_<const void*>(s), os);
}
inline void PrintTo(unsigned char* s, ::std::ostream* os) {
PrintTo(ImplicitCast_<const void*>(s), os);
}
__attribute__((visibility("default"))) void PrintTo(const char16_t* s, ::std::ostream* os);
inline void PrintTo(char16_t* s, ::std::ostream* os) {
PrintTo(ImplicitCast_<const char16_t*>(s), os);
}
__attribute__((visibility("default"))) void PrintTo(const char32_t* s, ::std::ostream* os);
inline void PrintTo(char32_t* s, ::std::ostream* os) {
PrintTo(ImplicitCast_<const char32_t*>(s), os);
}
__attribute__((visibility("default"))) void PrintTo(const wchar_t* s, ::std::ostream* os);
inline void PrintTo(wchar_t* s, ::std::ostream* os) {
PrintTo(ImplicitCast_<const wchar_t*>(s), os);
}
template <typename T>
void PrintRawArrayTo(const T a[], size_t count, ::std::ostream* os) {
UniversalPrint(a[0], os);
for (size_t i = 1; i != count; i++) {
*os << ", ";
UniversalPrint(a[i], os);
}
}
__attribute__((visibility("default"))) void PrintStringTo(const ::std::string& s, ::std::ostream* os);
inline void PrintTo(const ::std::string& s, ::std::ostream* os) {
PrintStringTo(s, os);
}
__attribute__((visibility("default"))) void PrintU16StringTo(const ::std::u16string& s, ::std::ostream* os);
inline void PrintTo(const ::std::u16string& s, ::std::ostream* os) {
PrintU16StringTo(s, os);
}
__attribute__((visibility("default"))) void PrintU32StringTo(const ::std::u32string& s, ::std::ostream* os);
inline void PrintTo(const ::std::u32string& s, ::std::ostream* os) {
PrintU32StringTo(s, os);
}
__attribute__((visibility("default"))) void PrintWideStringTo(const ::std::wstring& s, ::std::ostream* os);
inline void PrintTo(const ::std::wstring& s, ::std::ostream* os) {
PrintWideStringTo(s, os);
}
inline void PrintTo(internal::StringView sp, ::std::ostream* os) {
PrintTo(::std::string(sp), os);
}
inline void PrintTo(std::nullptr_t, ::std::ostream* os) { *os << "(nullptr)"; }
inline void PrintTo(const std::type_info& info, std::ostream* os) {
*os << internal::GetTypeName(info);
}
template <typename T>
void PrintTo(std::reference_wrapper<T> ref, ::std::ostream* os) {
UniversalPrinter<T&>::Print(ref.get(), os);
}
inline const void* VoidifyPointer(const void* p) { return p; }
inline const void* VoidifyPointer(volatile const void* p) {
return const_cast<const void*>(p);
}
template <typename T, typename Ptr>
void PrintSmartPointer(const Ptr& ptr, std::ostream* os, char) {
if (ptr == nullptr) {
*os << "(nullptr)";
} else {
*os << "(" << (VoidifyPointer)(ptr.get()) << ")";
}
}
template <typename T, typename Ptr,
typename = typename std::enable_if<!std::is_void<T>::value &&
!std::is_array<T>::value>::type>
void PrintSmartPointer(const Ptr& ptr, std::ostream* os, int) {
if (ptr == nullptr) {
*os << "(nullptr)";
} else {
*os << "(ptr = " << (VoidifyPointer)(ptr.get()) << ", value = ";
UniversalPrinter<T>::Print(*ptr, os);
*os << ")";
}
}
template <typename T, typename D>
void PrintTo(const std::unique_ptr<T, D>& ptr, std::ostream* os) {
(PrintSmartPointer<T>)(ptr, os, 0);
}
template <typename T>
void PrintTo(const std::shared_ptr<T>& ptr, std::ostream* os) {
(PrintSmartPointer<T>)(ptr, os, 0);
}
template <typename T>
void PrintTupleTo(const T&, std::integral_constant<size_t, 0>,
::std::ostream*) {}
template <typename T, size_t I>
void PrintTupleTo(const T& t, std::integral_constant<size_t, I>,
::std::ostream* os) {
PrintTupleTo(t, std::integral_constant<size_t, I - 1>(), os);
if (I > 1) {
*os << ", ";
}
UniversalPrinter<typename std::tuple_element<I - 1, T>::type>::Print(
std::get<I - 1>(t), os);
}
template <typename... Types>
void PrintTo(const ::std::tuple<Types...>& t, ::std::ostream* os) {
*os << "(";
PrintTupleTo(t, std::integral_constant<size_t, sizeof...(Types)>(), os);
*os << ")";
}
template <typename T1, typename T2>
void PrintTo(const ::std::pair<T1, T2>& value, ::std::ostream* os) {
*os << '(';
UniversalPrinter<T1>::Print(value.first, os);
*os << ", ";
UniversalPrinter<T2>::Print(value.second, os);
*os << ')';
}
template <typename T>
class UniversalPrinter {
public:
static void Print(const T& value, ::std::ostream* os) {
PrintTo(value, os);
}
};
template <typename T>
class UniversalPrinter<const T> : public UniversalPrinter<T> {};
template <>
class UniversalPrinter<Any> {
public:
static void Print(const Any& value, ::std::ostream* os) {
if (value.has_value()) {
*os << "value of type " << GetTypeName(value);
} else {
*os << "no value";
}
}
private:
static std::string GetTypeName(const Any& value) {
return internal::GetTypeName(value.type());
}
};
template <typename T>
class UniversalPrinter<Optional<T>> {
public:
static void Print(const Optional<T>& value, ::std::ostream* os) {
*os << '(';
if (!value) {
*os << "nullopt";
} else {
UniversalPrint(*value, os);
}
*os << ')';
}
};
template <>
class UniversalPrinter<decltype(Nullopt())> {
public:
static void Print(decltype(Nullopt()), ::std::ostream* os) {
*os << "(nullopt)";
}
};
template <typename... T>
class UniversalPrinter<Variant<T...>> {
public:
static void Print(const Variant<T...>& value, ::std::ostream* os) {
*os << '(';
std::visit(Visitor{os, value.index()}, value);
*os << ')';
}
private:
struct Visitor {
template <typename U>
void operator()(const U& u) const {
*os << "'" << GetTypeName<U>() << "(index = " << index
<< ")' with value ";
UniversalPrint(u, os);
}
::std::ostream* os;
std::size_t index;
};
};
template <typename T>
void UniversalPrintArray(const T* begin, size_t len, ::std::ostream* os) {
if (len == 0) {
*os << "{}";
} else {
*os << "{ ";
const size_t kThreshold = 18;
const size_t kChunkSize = 8;
if (len <= kThreshold) {
PrintRawArrayTo(begin, len, os);
} else {
PrintRawArrayTo(begin, kChunkSize, os);
*os << ", ..., ";
PrintRawArrayTo(begin + len - kChunkSize, kChunkSize, os);
}
*os << " }";
}
}
__attribute__((visibility("default"))) void UniversalPrintArray(const char* begin, size_t len,
::std::ostream* os);
__attribute__((visibility("default"))) void UniversalPrintArray(const char16_t* begin, size_t len,
::std::ostream* os);
__attribute__((visibility("default"))) void UniversalPrintArray(const char32_t* begin, size_t len,
::std::ostream* os);
__attribute__((visibility("default"))) void UniversalPrintArray(const wchar_t* begin, size_t len,
::std::ostream* os);
template <typename T, size_t N>
class UniversalPrinter<T[N]> {
public:
static void Print(const T (&a)[N], ::std::ostream* os) {
UniversalPrintArray(a, N, os);
}
};
template <typename T>
class UniversalPrinter<T&> {
public:
static void Print(const T& value, ::std::ostream* os) {
*os << "@" << reinterpret_cast<const void*>(&value) << " ";
UniversalPrint(value, os);
}
};
template <typename T>
class UniversalTersePrinter {
public:
static void Print(const T& value, ::std::ostream* os) {
UniversalPrint(value, os);
}
};
template <typename T>
class UniversalTersePrinter<T&> {
public:
static void Print(const T& value, ::std::ostream* os) {
UniversalPrint(value, os);
}
};
template <typename T>
class UniversalTersePrinter<std::reference_wrapper<T>> {
public:
static void Print(std::reference_wrapper<T> value, ::std::ostream* os) {
UniversalTersePrinter<T>::Print(value.get(), os);
}
};
template <typename T, size_t N>
class UniversalTersePrinter<T[N]> {
public:
static void Print(const T (&value)[N], ::std::ostream* os) {
UniversalPrinter<T[N]>::Print(value, os);
}
};
template <>
class UniversalTersePrinter<const char*> {
public:
static void Print(const char* str, ::std::ostream* os) {
if (str == nullptr) {
*os << "NULL";
} else {
UniversalPrint(std::string(str), os);
}
}
};
template <>
class UniversalTersePrinter<char*> : public UniversalTersePrinter<const char*> {
};
template <>
class UniversalTersePrinter<const char16_t*> {
public:
static void Print(const char16_t* str, ::std::ostream* os) {
if (str == nullptr) {
*os << "NULL";
} else {
UniversalPrint(::std::u16string(str), os);
}
}
};
template <>
class UniversalTersePrinter<char16_t*>
: public UniversalTersePrinter<const char16_t*> {};
template <>
class UniversalTersePrinter<const char32_t*> {
public:
static void Print(const char32_t* str, ::std::ostream* os) {
if (str == nullptr) {
*os << "NULL";
} else {
UniversalPrint(::std::u32string(str), os);
}
}
};
template <>
class UniversalTersePrinter<char32_t*>
: public UniversalTersePrinter<const char32_t*> {};
template <>
class UniversalTersePrinter<const wchar_t*> {
public:
static void Print(const wchar_t* str, ::std::ostream* os) {
if (str == nullptr) {
*os << "NULL";
} else {
UniversalPrint(::std::wstring(str), os);
}
}
};
template <>
class UniversalTersePrinter<wchar_t*> {
public:
static void Print(wchar_t* str, ::std::ostream* os) {
UniversalTersePrinter<const wchar_t*>::Print(str, os);
}
};
template <typename T>
void UniversalTersePrint(const T& value, ::std::ostream* os) {
UniversalTersePrinter<T>::Print(value, os);
}
template <typename T>
void UniversalPrint(const T& value, ::std::ostream* os) {
typedef T T1;
UniversalPrinter<T1>::Print(value, os);
}
typedef ::std::vector<::std::string> Strings;
template <typename Tuple>
void TersePrintPrefixToStrings(const Tuple&, std::integral_constant<size_t, 0>,
Strings*) {}
template <typename Tuple, size_t I>
void TersePrintPrefixToStrings(const Tuple& t,
std::integral_constant<size_t, I>,
Strings* strings) {
TersePrintPrefixToStrings(t, std::integral_constant<size_t, I - 1>(),
strings);
::std::stringstream ss;
UniversalTersePrint(std::get<I - 1>(t), &ss);
strings->push_back(ss.str());
}
template <typename Tuple>
Strings UniversalTersePrintTupleFieldsToStrings(const Tuple& value) {
Strings result;
TersePrintPrefixToStrings(
value, std::integral_constant<size_t, std::tuple_size<Tuple>::value>(),
&result);
return result;
}
}
template <typename T>
::std::string PrintToString(const T& value) {
::std::stringstream ss;
internal::UniversalTersePrinter<T>::Print(value, &ss);
return ss.str();
}
}
namespace testing {
class MatchResultListener {
public:
explicit MatchResultListener(::std::ostream* os) : stream_(os) {}
virtual ~MatchResultListener() = 0;
template <typename T>
MatchResultListener& operator<<(const T& x) {
if (stream_ != nullptr) *stream_ << x;
return *this;
}
::std::ostream* stream() { return stream_; }
bool IsInterested() const { return stream_ != nullptr; }
private:
::std::ostream* const stream_;
MatchResultListener(const MatchResultListener&) = delete;
MatchResultListener& operator=(const MatchResultListener&) = delete;
};
inline MatchResultListener::~MatchResultListener() {}
class __attribute__((visibility("default"))) MatcherDescriberInterface {
public:
virtual ~MatcherDescriberInterface() {}
virtual void DescribeTo(::std::ostream* os) const = 0;
virtual void DescribeNegationTo(::std::ostream* os) const {
*os << "not (";
DescribeTo(os);
*os << ")";
}
};
template <typename T>
class MatcherInterface : public MatcherDescriberInterface {
public:
virtual bool MatchAndExplain(T x, MatchResultListener* listener) const = 0;
};
namespace internal {
struct AnyEq {
template <typename A, typename B>
bool operator()(const A& a, const B& b) const {
return a == b;
}
};
struct AnyNe {
template <typename A, typename B>
bool operator()(const A& a, const B& b) const {
return a != b;
}
};
struct AnyLt {
template <typename A, typename B>
bool operator()(const A& a, const B& b) const {
return a < b;
}
};
struct AnyGt {
template <typename A, typename B>
bool operator()(const A& a, const B& b) const {
return a > b;
}
};
struct AnyLe {
template <typename A, typename B>
bool operator()(const A& a, const B& b) const {
return a <= b;
}
};
struct AnyGe {
template <typename A, typename B>
bool operator()(const A& a, const B& b) const {
return a >= b;
}
};
class DummyMatchResultListener : public MatchResultListener {
public:
DummyMatchResultListener() : MatchResultListener(nullptr) {}
private:
DummyMatchResultListener(const DummyMatchResultListener&) = delete;
DummyMatchResultListener& operator=(const DummyMatchResultListener&) = delete;
};
class StreamMatchResultListener : public MatchResultListener {
public:
explicit StreamMatchResultListener(::std::ostream* os)
: MatchResultListener(os) {}
private:
StreamMatchResultListener(const StreamMatchResultListener&) = delete;
StreamMatchResultListener& operator=(const StreamMatchResultListener&) =
delete;
};
struct SharedPayloadBase {
std::atomic<int> ref{1};
void Ref() { ref.fetch_add(1, std::memory_order_relaxed); }
bool Unref() { return ref.fetch_sub(1, std::memory_order_acq_rel) == 1; }
};
template <typename T>
struct SharedPayload : SharedPayloadBase {
explicit SharedPayload(const T& v) : value(v) {}
explicit SharedPayload(T&& v) : value(std::move(v)) {}
static void Destroy(SharedPayloadBase* shared) {
delete static_cast<SharedPayload*>(shared);
}
T value;
};
template <typename T>
class MatcherBase : private MatcherDescriberInterface {
public:
bool MatchAndExplain(const T& x, MatchResultListener* listener) const {
switch (0) case 0: default: if (::testing::internal::IsTrue(vtable_ != nullptr)) ; else ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/gtest-matchers.h", 269) .GetStream() << "Condition " "vtable_ != nullptr" " failed. ";
return vtable_->match_and_explain(*this, x, listener);
}
bool Matches(const T& x) const {
DummyMatchResultListener dummy;
return MatchAndExplain(x, &dummy);
}
void DescribeTo(::std::ostream* os) const final {
switch (0) case 0: default: if (::testing::internal::IsTrue(vtable_ != nullptr)) ; else ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/gtest-matchers.h", 281) .GetStream() << "Condition " "vtable_ != nullptr" " failed. ";
vtable_->describe(*this, os, false);
}
void DescribeNegationTo(::std::ostream* os) const final {
switch (0) case 0: default: if (::testing::internal::IsTrue(vtable_ != nullptr)) ; else ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/gtest-matchers.h", 287) .GetStream() << "Condition " "vtable_ != nullptr" " failed. ";
vtable_->describe(*this, os, true);
}
void ExplainMatchResultTo(const T& x, ::std::ostream* os) const {
StreamMatchResultListener listener(os);
MatchAndExplain(x, &listener);
}
const MatcherDescriberInterface* GetDescriber() const {
if (vtable_ == nullptr) return nullptr;
return vtable_->get_describer(*this);
}
protected:
MatcherBase() : vtable_(nullptr), buffer_() {}
template <typename U>
explicit MatcherBase(const MatcherInterface<U>* impl)
: vtable_(nullptr), buffer_() {
Init(impl);
}
template <typename M, typename = typename std::remove_reference<
M>::type::is_gtest_matcher>
MatcherBase(M&& m) : vtable_(nullptr), buffer_() {
Init(std::forward<M>(m));
}
MatcherBase(const MatcherBase& other)
: vtable_(other.vtable_), buffer_(other.buffer_) {
if (IsShared()) buffer_.shared->Ref();
}
MatcherBase& operator=(const MatcherBase& other) {
if (this == &other) return *this;
Destroy();
vtable_ = other.vtable_;
buffer_ = other.buffer_;
if (IsShared()) buffer_.shared->Ref();
return *this;
}
MatcherBase(MatcherBase&& other)
: vtable_(other.vtable_), buffer_(other.buffer_) {
other.vtable_ = nullptr;
}
MatcherBase& operator=(MatcherBase&& other) {
if (this == &other) return *this;
Destroy();
vtable_ = other.vtable_;
buffer_ = other.buffer_;
other.vtable_ = nullptr;
return *this;
}
~MatcherBase() override { Destroy(); }
private:
struct VTable {
bool (*match_and_explain)(const MatcherBase&, const T&,
MatchResultListener*);
void (*describe)(const MatcherBase&, std::ostream*, bool negation);
const MatcherDescriberInterface* (*get_describer)(const MatcherBase&);
void (*shared_destroy)(SharedPayloadBase*);
};
bool IsShared() const {
return vtable_ != nullptr && vtable_->shared_destroy != nullptr;
}
template <typename P>
static auto MatchAndExplainImpl(const MatcherBase& m, const T& value,
MatchResultListener* listener)
-> decltype(P::Get(m).MatchAndExplain(value, listener->stream())) {
return P::Get(m).MatchAndExplain(value, listener->stream());
}
template <typename P>
static auto MatchAndExplainImpl(const MatcherBase& m, const T& value,
MatchResultListener* listener)
-> decltype(P::Get(m).MatchAndExplain(value, listener)) {
return P::Get(m).MatchAndExplain(value, listener);
}
template <typename P>
static void DescribeImpl(const MatcherBase& m, std::ostream* os,
bool negation) {
if (negation) {
P::Get(m).DescribeNegationTo(os);
} else {
P::Get(m).DescribeTo(os);
}
}
template <typename P>
static const MatcherDescriberInterface* GetDescriberImpl(
const MatcherBase& m) {
return std::get<(
std::is_convertible<decltype(&P::Get(m)),
const MatcherDescriberInterface*>::value
? 1
: 0)>(std::make_tuple(&m, &P::Get(m)));
}
template <typename P>
const VTable* GetVTable() {
static constexpr VTable kVTable = {&MatchAndExplainImpl<P>,
&DescribeImpl<P>, &GetDescriberImpl<P>,
P::shared_destroy};
return &kVTable;
}
union Buffer {
void* ptr;
double d;
int64_t i;
SharedPayloadBase* shared;
};
void Destroy() {
if (IsShared() && buffer_.shared->Unref()) {
vtable_->shared_destroy(buffer_.shared);
}
}
template <typename M>
static constexpr bool IsInlined() {
return sizeof(M) <= sizeof(Buffer) && alignof(M) <= alignof(Buffer) &&
std::is_trivially_copy_constructible<M>::value &&
std::is_trivially_destructible<M>::value;
}
template <typename M, bool = MatcherBase::IsInlined<M>()>
struct ValuePolicy {
static const M& Get(const MatcherBase& m) {
const M* ptr =
static_cast<const M*>(static_cast<const void*>(&m.buffer_));
return *ptr;
}
static void Init(MatcherBase& m, M impl) {
::new (static_cast<void*>(&m.buffer_)) M(impl);
}
static constexpr auto shared_destroy = nullptr;
};
template <typename M>
struct ValuePolicy<M, false> {
using Shared = SharedPayload<M>;
static const M& Get(const MatcherBase& m) {
return static_cast<Shared*>(m.buffer_.shared)->value;
}
template <typename Arg>
static void Init(MatcherBase& m, Arg&& arg) {
m.buffer_.shared = new Shared(std::forward<Arg>(arg));
}
static constexpr auto shared_destroy = &Shared::Destroy;
};
template <typename U, bool B>
struct ValuePolicy<const MatcherInterface<U>*, B> {
using M = const MatcherInterface<U>;
using Shared = SharedPayload<std::unique_ptr<M>>;
static const M& Get(const MatcherBase& m) {
return *static_cast<Shared*>(m.buffer_.shared)->value;
}
static void Init(MatcherBase& m, M* impl) {
m.buffer_.shared = new Shared(std::unique_ptr<M>(impl));
}
static constexpr auto shared_destroy = &Shared::Destroy;
};
template <typename M>
void Init(M&& m) {
using MM = typename std::decay<M>::type;
using Policy = ValuePolicy<MM>;
vtable_ = GetVTable<Policy>();
Policy::Init(*this, std::forward<M>(m));
}
const VTable* vtable_;
Buffer buffer_;
};
}
template <typename T>
class Matcher : public internal::MatcherBase<T> {
public:
explicit Matcher() {}
explicit Matcher(const MatcherInterface<const T&>* impl)
: internal::MatcherBase<T>(impl) {}
template <typename U>
explicit Matcher(
const MatcherInterface<U>* impl,
typename std::enable_if<!std::is_same<U, const U&>::value>::type* =
nullptr)
: internal::MatcherBase<T>(impl) {}
template <typename M, typename = typename std::remove_reference<
M>::type::is_gtest_matcher>
Matcher(M&& m) : internal::MatcherBase<T>(std::forward<M>(m)) {}
Matcher(T value);
};
template <>
class __attribute__((visibility("default"))) Matcher<const std::string&>
: public internal::MatcherBase<const std::string&> {
public:
Matcher() {}
explicit Matcher(const MatcherInterface<const std::string&>* impl)
: internal::MatcherBase<const std::string&>(impl) {}
template <typename M, typename = typename std::remove_reference<
M>::type::is_gtest_matcher>
Matcher(M&& m)
: internal::MatcherBase<const std::string&>(std::forward<M>(m)) {}
Matcher(const std::string& s);
Matcher(const char* s);
};
template <>
class __attribute__((visibility("default"))) Matcher<std::string>
: public internal::MatcherBase<std::string> {
public:
Matcher() {}
explicit Matcher(const MatcherInterface<const std::string&>* impl)
: internal::MatcherBase<std::string>(impl) {}
explicit Matcher(const MatcherInterface<std::string>* impl)
: internal::MatcherBase<std::string>(impl) {}
template <typename M, typename = typename std::remove_reference<
M>::type::is_gtest_matcher>
Matcher(M&& m)
: internal::MatcherBase<std::string>(std::forward<M>(m)) {}
Matcher(const std::string& s);
Matcher(const char* s);
};
template <>
class __attribute__((visibility("default"))) Matcher<const internal::StringView&>
: public internal::MatcherBase<const internal::StringView&> {
public:
Matcher() {}
explicit Matcher(const MatcherInterface<const internal::StringView&>* impl)
: internal::MatcherBase<const internal::StringView&>(impl) {}
template <typename M, typename = typename std::remove_reference<
M>::type::is_gtest_matcher>
Matcher(M&& m)
: internal::MatcherBase<const internal::StringView&>(std::forward<M>(m)) {
}
Matcher(const std::string& s);
Matcher(const char* s);
Matcher(internal::StringView s);
};
template <>
class __attribute__((visibility("default"))) Matcher<internal::StringView>
: public internal::MatcherBase<internal::StringView> {
public:
Matcher() {}
explicit Matcher(const MatcherInterface<const internal::StringView&>* impl)
: internal::MatcherBase<internal::StringView>(impl) {}
explicit Matcher(const MatcherInterface<internal::StringView>* impl)
: internal::MatcherBase<internal::StringView>(impl) {}
template <typename M, typename = typename std::remove_reference<
M>::type::is_gtest_matcher>
Matcher(M&& m)
: internal::MatcherBase<internal::StringView>(std::forward<M>(m)) {}
Matcher(const std::string& s);
Matcher(const char* s);
Matcher(internal::StringView s);
};
template <typename T>
std::ostream& operator<<(std::ostream& os, const Matcher<T>& matcher) {
matcher.DescribeTo(&os);
return os;
}
template <class Impl>
class PolymorphicMatcher {
public:
explicit PolymorphicMatcher(const Impl& an_impl) : impl_(an_impl) {}
Impl& mutable_impl() { return impl_; }
const Impl& impl() const { return impl_; }
template <typename T>
operator Matcher<T>() const {
return Matcher<T>(new MonomorphicImpl<const T&>(impl_));
}
private:
template <typename T>
class MonomorphicImpl : public MatcherInterface<T> {
public:
explicit MonomorphicImpl(const Impl& impl) : impl_(impl) {}
void DescribeTo(::std::ostream* os) const override { impl_.DescribeTo(os); }
void DescribeNegationTo(::std::ostream* os) const override {
impl_.DescribeNegationTo(os);
}
bool MatchAndExplain(T x, MatchResultListener* listener) const override {
return impl_.MatchAndExplain(x, listener);
}
private:
const Impl impl_;
};
Impl impl_;
};
template <typename T>
inline Matcher<T> MakeMatcher(const MatcherInterface<T>* impl) {
return Matcher<T>(impl);
}
template <class Impl>
inline PolymorphicMatcher<Impl> MakePolymorphicMatcher(const Impl& impl) {
return PolymorphicMatcher<Impl>(impl);
}
namespace internal {
template <typename D, typename Rhs, typename Op>
class ComparisonBase {
public:
explicit ComparisonBase(const Rhs& rhs) : rhs_(rhs) {}
using is_gtest_matcher = void;
template <typename Lhs>
bool MatchAndExplain(const Lhs& lhs, std::ostream*) const {
return Op()(lhs, Unwrap(rhs_));
}
void DescribeTo(std::ostream* os) const {
*os << D::Desc() << " ";
UniversalPrint(Unwrap(rhs_), os);
}
void DescribeNegationTo(std::ostream* os) const {
*os << D::NegatedDesc() << " ";
UniversalPrint(Unwrap(rhs_), os);
}
private:
template <typename T>
static const T& Unwrap(const T& v) {
return v;
}
template <typename T>
static const T& Unwrap(std::reference_wrapper<T> v) {
return v;
}
Rhs rhs_;
};
template <typename Rhs>
class EqMatcher : public ComparisonBase<EqMatcher<Rhs>, Rhs, AnyEq> {
public:
explicit EqMatcher(const Rhs& rhs)
: ComparisonBase<EqMatcher<Rhs>, Rhs, AnyEq>(rhs) {}
static const char* Desc() { return "is equal to"; }
static const char* NegatedDesc() { return "isn't equal to"; }
};
template <typename Rhs>
class NeMatcher : public ComparisonBase<NeMatcher<Rhs>, Rhs, AnyNe> {
public:
explicit NeMatcher(const Rhs& rhs)
: ComparisonBase<NeMatcher<Rhs>, Rhs, AnyNe>(rhs) {}
static const char* Desc() { return "isn't equal to"; }
static const char* NegatedDesc() { return "is equal to"; }
};
template <typename Rhs>
class LtMatcher : public ComparisonBase<LtMatcher<Rhs>, Rhs, AnyLt> {
public:
explicit LtMatcher(const Rhs& rhs)
: ComparisonBase<LtMatcher<Rhs>, Rhs, AnyLt>(rhs) {}
static const char* Desc() { return "is <"; }
static const char* NegatedDesc() { return "isn't <"; }
};
template <typename Rhs>
class GtMatcher : public ComparisonBase<GtMatcher<Rhs>, Rhs, AnyGt> {
public:
explicit GtMatcher(const Rhs& rhs)
: ComparisonBase<GtMatcher<Rhs>, Rhs, AnyGt>(rhs) {}
static const char* Desc() { return "is >"; }
static const char* NegatedDesc() { return "isn't >"; }
};
template <typename Rhs>
class LeMatcher : public ComparisonBase<LeMatcher<Rhs>, Rhs, AnyLe> {
public:
explicit LeMatcher(const Rhs& rhs)
: ComparisonBase<LeMatcher<Rhs>, Rhs, AnyLe>(rhs) {}
static const char* Desc() { return "is <="; }
static const char* NegatedDesc() { return "isn't <="; }
};
template <typename Rhs>
class GeMatcher : public ComparisonBase<GeMatcher<Rhs>, Rhs, AnyGe> {
public:
explicit GeMatcher(const Rhs& rhs)
: ComparisonBase<GeMatcher<Rhs>, Rhs, AnyGe>(rhs) {}
static const char* Desc() { return "is >="; }
static const char* NegatedDesc() { return "isn't >="; }
};
template <typename T, typename = typename std::enable_if<
std::is_constructible<std::string, T>::value>::type>
using StringLike = T;
class MatchesRegexMatcher {
public:
MatchesRegexMatcher(const RE* regex, bool full_match)
: regex_(regex), full_match_(full_match) {}
bool MatchAndExplain(const internal::StringView& s,
MatchResultListener* listener) const {
return MatchAndExplain(std::string(s), listener);
}
template <typename CharType>
bool MatchAndExplain(CharType* s, MatchResultListener* listener) const {
return s != nullptr && MatchAndExplain(std::string(s), listener);
}
template <class MatcheeStringType>
bool MatchAndExplain(const MatcheeStringType& s,
MatchResultListener* ) const {
const std::string s2(s);
return full_match_ ? RE::FullMatch(s2, *regex_)
: RE::PartialMatch(s2, *regex_);
}
void DescribeTo(::std::ostream* os) const {
*os << (full_match_ ? "matches" : "contains") << " regular expression ";
UniversalPrinter<std::string>::Print(regex_->pattern(), os);
}
void DescribeNegationTo(::std::ostream* os) const {
*os << "doesn't " << (full_match_ ? "match" : "contain")
<< " regular expression ";
UniversalPrinter<std::string>::Print(regex_->pattern(), os);
}
private:
const std::shared_ptr<const RE> regex_;
const bool full_match_;
};
}
inline PolymorphicMatcher<internal::MatchesRegexMatcher> MatchesRegex(
const internal::RE* regex) {
return MakePolymorphicMatcher(internal::MatchesRegexMatcher(regex, true));
}
template <typename T = std::string>
PolymorphicMatcher<internal::MatchesRegexMatcher> MatchesRegex(
const internal::StringLike<T>& regex) {
return MatchesRegex(new internal::RE(std::string(regex)));
}
inline PolymorphicMatcher<internal::MatchesRegexMatcher> ContainsRegex(
const internal::RE* regex) {
return MakePolymorphicMatcher(internal::MatchesRegexMatcher(regex, false));
}
template <typename T = std::string>
PolymorphicMatcher<internal::MatchesRegexMatcher> ContainsRegex(
const internal::StringLike<T>& regex) {
return ContainsRegex(new internal::RE(std::string(regex)));
}
template <typename T>
inline internal::EqMatcher<T> Eq(T x) {
return internal::EqMatcher<T>(x);
}
template <typename T>
Matcher<T>::Matcher(T value) {
*this = Eq(value);
}
template <typename Lhs, typename Rhs>
inline Matcher<Lhs> TypedEq(const Rhs& rhs) {
return Eq(rhs);
}
template <typename Rhs>
inline internal::GeMatcher<Rhs> Ge(Rhs x) {
return internal::GeMatcher<Rhs>(x);
}
template <typename Rhs>
inline internal::GtMatcher<Rhs> Gt(Rhs x) {
return internal::GtMatcher<Rhs>(x);
}
template <typename Rhs>
inline internal::LeMatcher<Rhs> Le(Rhs x) {
return internal::LeMatcher<Rhs>(x);
}
template <typename Rhs>
inline internal::LtMatcher<Rhs> Lt(Rhs x) {
return internal::LtMatcher<Rhs>(x);
}
template <typename Rhs>
inline internal::NeMatcher<Rhs> Ne(Rhs x) {
return internal::NeMatcher<Rhs>(x);
}
}
namespace testing { __attribute__((visibility("default"))) extern ::std::string FLAGS_gtest_internal_run_death_test; } static_assert(true, "no-op to require trailing semicolon");
namespace testing {
namespace internal {
const char kDeathTestStyleFlag[] = "death_test_style";
const char kDeathTestUseFork[] = "death_test_use_fork";
const char kInternalRunDeathTestFlag[] = "internal_run_death_test";
class __attribute__((visibility("default"))) DeathTest {
public:
static bool Create(const char* statement, Matcher<const std::string&> matcher,
const char* file, int line, DeathTest** test);
DeathTest();
virtual ~DeathTest() {}
class ReturnSentinel {
public:
explicit ReturnSentinel(DeathTest* test) : test_(test) {}
~ReturnSentinel() { test_->Abort(TEST_ENCOUNTERED_RETURN_STATEMENT); }
private:
DeathTest* const test_;
ReturnSentinel(const ReturnSentinel&) = delete;
ReturnSentinel& operator=(const ReturnSentinel&) = delete;
};
enum TestRole { OVERSEE_TEST, EXECUTE_TEST };
enum AbortReason {
TEST_ENCOUNTERED_RETURN_STATEMENT,
TEST_THREW_EXCEPTION,
TEST_DID_NOT_DIE
};
virtual TestRole AssumeRole() = 0;
virtual int Wait() = 0;
virtual bool Passed(bool exit_status_ok) = 0;
virtual void Abort(AbortReason reason) = 0;
static const char* LastMessage();
static void set_last_death_test_message(const std::string& message);
private:
static std::string last_death_test_message_;
DeathTest(const DeathTest&) = delete;
DeathTest& operator=(const DeathTest&) = delete;
};
class DeathTestFactory {
public:
virtual ~DeathTestFactory() {}
virtual bool Create(const char* statement,
Matcher<const std::string&> matcher, const char* file,
int line, DeathTest** test) = 0;
};
class DefaultDeathTestFactory : public DeathTestFactory {
public:
bool Create(const char* statement, Matcher<const std::string&> matcher,
const char* file, int line, DeathTest** test) override;
};
__attribute__((visibility("default"))) bool ExitedUnsuccessfully(int exit_status);
inline Matcher<const ::std::string&> MakeDeathTestMatcher(
::testing::internal::RE regex) {
return ContainsRegex(regex.pattern());
}
inline Matcher<const ::std::string&> MakeDeathTestMatcher(const char* regex) {
return ContainsRegex(regex);
}
inline Matcher<const ::std::string&> MakeDeathTestMatcher(
const ::std::string& regex) {
return ContainsRegex(regex);
}
inline Matcher<const ::std::string&> MakeDeathTestMatcher(
Matcher<const ::std::string&> matcher) {
return matcher;
}
class InternalRunDeathTestFlag {
public:
InternalRunDeathTestFlag(const std::string& a_file, int a_line, int an_index,
int a_write_fd)
: file_(a_file), line_(a_line), index_(an_index), write_fd_(a_write_fd) {}
~InternalRunDeathTestFlag() {
if (write_fd_ >= 0) posix::Close(write_fd_);
}
const std::string& file() const { return file_; }
int line() const { return line_; }
int index() const { return index_; }
int write_fd() const { return write_fd_; }
private:
std::string file_;
int line_;
int index_;
int write_fd_;
InternalRunDeathTestFlag(const InternalRunDeathTestFlag&) = delete;
InternalRunDeathTestFlag& operator=(const InternalRunDeathTestFlag&) = delete;
};
InternalRunDeathTestFlag* ParseInternalRunDeathTestFlag();
}
}
namespace testing { __attribute__((visibility("default"))) extern ::std::string FLAGS_gtest_death_test_style; } static_assert(true, "no-op to require trailing semicolon");
namespace testing {
namespace internal {
__attribute__((visibility("default"))) bool InDeathTestChild();
}
class __attribute__((visibility("default"))) ExitedWithCode {
public:
explicit ExitedWithCode(int exit_code);
ExitedWithCode(const ExitedWithCode&) = default;
void operator=(const ExitedWithCode& other) = delete;
bool operator()(int exit_status) const;
private:
const int exit_code_;
};
class __attribute__((visibility("default"))) KilledBySignal {
public:
explicit KilledBySignal(int signum);
bool operator()(int exit_status) const;
private:
const int signum_;
};
}
namespace std __attribute__ ((__visibility__ ("default")))
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
template<typename _Tp, typename _CharT = char,
typename _Traits = char_traits<_CharT>, typename _Dist = ptrdiff_t>
class istream_iterator
: public iterator<input_iterator_tag, _Tp, _Dist, const _Tp*, const _Tp&>
{
public:
typedef _CharT char_type;
typedef _Traits traits_type;
typedef basic_istream<_CharT, _Traits> istream_type;
private:
istream_type* _M_stream;
_Tp _M_value;
bool _M_ok;
public:
constexpr istream_iterator()
noexcept(is_nothrow_default_constructible<_Tp>::value)
: _M_stream(0), _M_value(), _M_ok(false) {}
istream_iterator(istream_type& __s)
: _M_stream(std::__addressof(__s)), _M_ok(true)
{ _M_read(); }
constexpr
istream_iterator(const istream_iterator& __obj)
noexcept(is_nothrow_copy_constructible<_Tp>::value)
: _M_stream(__obj._M_stream), _M_value(__obj._M_value),
_M_ok(__obj._M_ok)
{ }
istream_iterator& operator=(const istream_iterator&) = default;
~istream_iterator() = default;
[[__nodiscard__]]
const _Tp&
operator*() const noexcept
{
;
return _M_value;
}
[[__nodiscard__]]
const _Tp*
operator->() const noexcept
{ return std::__addressof((operator*())); }
istream_iterator&
operator++()
{
;
_M_read();
return *this;
}
istream_iterator
operator++(int)
{
;
istream_iterator __tmp = *this;
_M_read();
return __tmp;
}
private:
bool
_M_equal(const istream_iterator& __x) const noexcept
{
return (_M_ok == __x._M_ok) && (!_M_ok || _M_stream == __x._M_stream);
}
void
_M_read()
{
if (_M_stream && !(*_M_stream >> _M_value))
{
_M_stream = 0;
_M_ok = false;
}
}
[[__nodiscard__]]
friend bool
operator==(const istream_iterator& __x, const istream_iterator& __y)
noexcept
{ return __x._M_equal(__y); }
[[__nodiscard__]]
friend bool
operator!=(const istream_iterator& __x, const istream_iterator& __y)
noexcept
{ return !__x._M_equal(__y); }
};
template<typename _Tp, typename _CharT = char,
typename _Traits = char_traits<_CharT> >
class ostream_iterator
: public iterator<output_iterator_tag, void, void, void, void>
{
public:
typedef _CharT char_type;
typedef _Traits traits_type;
typedef basic_ostream<_CharT, _Traits> ostream_type;
private:
ostream_type* _M_stream;
const _CharT* _M_string;
public:
ostream_iterator(ostream_type& __s) noexcept
: _M_stream(std::__addressof(__s)), _M_string(0) {}
ostream_iterator(ostream_type& __s, const _CharT* __c) noexcept
: _M_stream(std::__addressof(__s)), _M_string(__c) { }
ostream_iterator(const ostream_iterator& __obj) noexcept
: _M_stream(__obj._M_stream), _M_string(__obj._M_string) { }
ostream_iterator& operator=(const ostream_iterator&) = default;
ostream_iterator&
operator=(const _Tp& __value)
{
;
*_M_stream << __value;
if (_M_string)
*_M_stream << _M_string;
return *this;
}
[[__nodiscard__]]
ostream_iterator&
operator*() noexcept
{ return *this; }
ostream_iterator&
operator++() noexcept
{ return *this; }
ostream_iterator&
operator++(int) noexcept
{ return *this; }
};
#pragma GCC diagnostic pop
}
namespace testing {
class __attribute__((visibility("default"))) TestPartResult {
public:
enum Type {
kSuccess,
kNonFatalFailure,
kFatalFailure,
kSkip
};
TestPartResult(Type a_type, const char* a_file_name, int a_line_number,
const char* a_message)
: type_(a_type),
file_name_(a_file_name == nullptr ? "" : a_file_name),
line_number_(a_line_number),
summary_(ExtractSummary(a_message)),
message_(a_message) {}
Type type() const { return type_; }
const char* file_name() const {
return file_name_.empty() ? nullptr : file_name_.c_str();
}
int line_number() const { return line_number_; }
const char* summary() const { return summary_.c_str(); }
const char* message() const { return message_.c_str(); }
bool skipped() const { return type_ == kSkip; }
bool passed() const { return type_ == kSuccess; }
bool nonfatally_failed() const { return type_ == kNonFatalFailure; }
bool fatally_failed() const { return type_ == kFatalFailure; }
bool failed() const { return fatally_failed() || nonfatally_failed(); }
private:
Type type_;
static std::string ExtractSummary(const char* message);
std::string file_name_;
int line_number_;
std::string summary_;
std::string message_;
};
std::ostream& operator<<(std::ostream& os, const TestPartResult& result);
class __attribute__((visibility("default"))) TestPartResultArray {
public:
TestPartResultArray() {}
void Append(const TestPartResult& result);
const TestPartResult& GetTestPartResult(int index) const;
int size() const;
private:
std::vector<TestPartResult> array_;
TestPartResultArray(const TestPartResultArray&) = delete;
TestPartResultArray& operator=(const TestPartResultArray&) = delete;
};
class __attribute__((visibility("default"))) TestPartResultReporterInterface {
public:
virtual ~TestPartResultReporterInterface() {}
virtual void ReportTestPartResult(const TestPartResult& result) = 0;
};
namespace internal {
class __attribute__((visibility("default"))) HasNewFatalFailureHelper
: public TestPartResultReporterInterface {
public:
HasNewFatalFailureHelper();
~HasNewFatalFailureHelper() override;
void ReportTestPartResult(const TestPartResult& result) override;
bool has_new_fatal_failure() const { return has_new_fatal_failure_; }
private:
bool has_new_fatal_failure_;
TestPartResultReporterInterface* original_reporter_;
HasNewFatalFailureHelper(const HasNewFatalFailureHelper&) = delete;
HasNewFatalFailureHelper& operator=(const HasNewFatalFailureHelper&) = delete;
};
}
}
namespace testing {
template <class ParamType>
struct TestParamInfo {
TestParamInfo(const ParamType& a_param, size_t an_index)
: param(a_param), index(an_index) {}
ParamType param;
size_t index;
};
struct PrintToStringParamName {
template <class ParamType>
std::string operator()(const TestParamInfo<ParamType>& info) const {
return PrintToString(info.param);
}
};
namespace internal {
__attribute__((visibility("default"))) void ReportInvalidTestSuiteType(const char* test_suite_name,
CodeLocation code_location);
template <typename>
class ParamGeneratorInterface;
template <typename>
class ParamGenerator;
template <typename T>
class ParamIteratorInterface {
public:
virtual ~ParamIteratorInterface() {}
virtual const ParamGeneratorInterface<T>* BaseGenerator() const = 0;
virtual void Advance() = 0;
virtual ParamIteratorInterface* Clone() const = 0;
virtual const T* Current() const = 0;
virtual bool Equals(const ParamIteratorInterface& other) const = 0;
};
template <typename T>
class ParamIterator {
public:
typedef T value_type;
typedef const T& reference;
typedef ptrdiff_t difference_type;
ParamIterator(const ParamIterator& other) : impl_(other.impl_->Clone()) {}
ParamIterator& operator=(const ParamIterator& other) {
if (this != &other) impl_.reset(other.impl_->Clone());
return *this;
}
const T& operator*() const { return *impl_->Current(); }
const T* operator->() const { return impl_->Current(); }
ParamIterator& operator++() {
impl_->Advance();
return *this;
}
ParamIterator operator++(int ) {
ParamIteratorInterface<T>* clone = impl_->Clone();
impl_->Advance();
return ParamIterator(clone);
}
bool operator==(const ParamIterator& other) const {
return impl_.get() == other.impl_.get() || impl_->Equals(*other.impl_);
}
bool operator!=(const ParamIterator& other) const {
return !(*this == other);
}
private:
friend class ParamGenerator<T>;
explicit ParamIterator(ParamIteratorInterface<T>* impl) : impl_(impl) {}
std::unique_ptr<ParamIteratorInterface<T>> impl_;
};
template <typename T>
class ParamGeneratorInterface {
public:
typedef T ParamType;
virtual ~ParamGeneratorInterface() {}
virtual ParamIteratorInterface<T>* Begin() const = 0;
virtual ParamIteratorInterface<T>* End() const = 0;
};
template <typename T>
class ParamGenerator {
public:
typedef ParamIterator<T> iterator;
explicit ParamGenerator(ParamGeneratorInterface<T>* impl) : impl_(impl) {}
ParamGenerator(const ParamGenerator& other) : impl_(other.impl_) {}
ParamGenerator& operator=(const ParamGenerator& other) {
impl_ = other.impl_;
return *this;
}
iterator begin() const { return iterator(impl_->Begin()); }
iterator end() const { return iterator(impl_->End()); }
private:
std::shared_ptr<const ParamGeneratorInterface<T>> impl_;
};
template <typename T, typename IncrementT>
class RangeGenerator : public ParamGeneratorInterface<T> {
public:
RangeGenerator(T begin, T end, IncrementT step)
: begin_(begin),
end_(end),
step_(step),
end_index_(CalculateEndIndex(begin, end, step)) {}
~RangeGenerator() override {}
ParamIteratorInterface<T>* Begin() const override {
return new Iterator(this, begin_, 0, step_);
}
ParamIteratorInterface<T>* End() const override {
return new Iterator(this, end_, end_index_, step_);
}
private:
class Iterator : public ParamIteratorInterface<T> {
public:
Iterator(const ParamGeneratorInterface<T>* base, T value, int index,
IncrementT step)
: base_(base), value_(value), index_(index), step_(step) {}
~Iterator() override {}
const ParamGeneratorInterface<T>* BaseGenerator() const override {
return base_;
}
void Advance() override {
value_ = static_cast<T>(value_ + step_);
index_++;
}
ParamIteratorInterface<T>* Clone() const override {
return new Iterator(*this);
}
const T* Current() const override { return &value_; }
bool Equals(const ParamIteratorInterface<T>& other) const override {
switch (0) case 0: default: if (::testing::internal::IsTrue(BaseGenerator() == other.BaseGenerator())) ; else ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-param-util.h", 249) .GetStream() << "Condition " "BaseGenerator() == other.BaseGenerator()" " failed. "
<< "The program attempted to compare iterators "
<< "from different generators." << std::endl;
const int other_index =
CheckedDowncastToActualType<const Iterator>(&other)->index_;
return index_ == other_index;
}
private:
Iterator(const Iterator& other)
: ParamIteratorInterface<T>(),
base_(other.base_),
value_(other.value_),
index_(other.index_),
step_(other.step_) {}
void operator=(const Iterator& other);
const ParamGeneratorInterface<T>* const base_;
T value_;
int index_;
const IncrementT step_;
};
static int CalculateEndIndex(const T& begin, const T& end,
const IncrementT& step) {
int end_index = 0;
for (T i = begin; i < end; i = static_cast<T>(i + step)) end_index++;
return end_index;
}
void operator=(const RangeGenerator& other);
const T begin_;
const T end_;
const IncrementT step_;
const int end_index_;
};
template <typename T>
class ValuesInIteratorRangeGenerator : public ParamGeneratorInterface<T> {
public:
template <typename ForwardIterator>
ValuesInIteratorRangeGenerator(ForwardIterator begin, ForwardIterator end)
: container_(begin, end) {}
~ValuesInIteratorRangeGenerator() override {}
ParamIteratorInterface<T>* Begin() const override {
return new Iterator(this, container_.begin());
}
ParamIteratorInterface<T>* End() const override {
return new Iterator(this, container_.end());
}
private:
typedef typename ::std::vector<T> ContainerType;
class Iterator : public ParamIteratorInterface<T> {
public:
Iterator(const ParamGeneratorInterface<T>* base,
typename ContainerType::const_iterator iterator)
: base_(base), iterator_(iterator) {}
~Iterator() override {}
const ParamGeneratorInterface<T>* BaseGenerator() const override {
return base_;
}
void Advance() override {
++iterator_;
value_.reset();
}
ParamIteratorInterface<T>* Clone() const override {
return new Iterator(*this);
}
const T* Current() const override {
if (value_.get() == nullptr) value_.reset(new T(*iterator_));
return value_.get();
}
bool Equals(const ParamIteratorInterface<T>& other) const override {
switch (0) case 0: default: if (::testing::internal::IsTrue(BaseGenerator() == other.BaseGenerator())) ; else ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-param-util.h", 345) .GetStream() << "Condition " "BaseGenerator() == other.BaseGenerator()" " failed. "
<< "The program attempted to compare iterators "
<< "from different generators." << std::endl;
return iterator_ ==
CheckedDowncastToActualType<const Iterator>(&other)->iterator_;
}
private:
Iterator(const Iterator& other)
: ParamIteratorInterface<T>(),
base_(other.base_),
iterator_(other.iterator_) {}
const ParamGeneratorInterface<T>* const base_;
typename ContainerType::const_iterator iterator_;
mutable std::unique_ptr<const T> value_;
};
void operator=(const ValuesInIteratorRangeGenerator& other);
const ContainerType container_;
};
template <class ParamType>
std::string DefaultParamName(const TestParamInfo<ParamType>& info) {
Message name_stream;
name_stream << info.index;
return name_stream.GetString();
}
template <typename T = int>
void TestNotEmpty() {
static_assert(sizeof(T) == 0, "Empty arguments are not allowed.");
}
template <typename T = int>
void TestNotEmpty(const T&) {}
template <class TestClass>
class ParameterizedTestFactory : public TestFactoryBase {
public:
typedef typename TestClass::ParamType ParamType;
explicit ParameterizedTestFactory(ParamType parameter)
: parameter_(parameter) {}
Test* CreateTest() override {
TestClass::SetParam(¶meter_);
return new TestClass();
}
private:
const ParamType parameter_;
ParameterizedTestFactory(const ParameterizedTestFactory&) = delete;
ParameterizedTestFactory& operator=(const ParameterizedTestFactory&) = delete;
};
template <class ParamType>
class TestMetaFactoryBase {
public:
virtual ~TestMetaFactoryBase() {}
virtual TestFactoryBase* CreateTestFactory(ParamType parameter) = 0;
};
template <class TestSuite>
class TestMetaFactory
: public TestMetaFactoryBase<typename TestSuite::ParamType> {
public:
using ParamType = typename TestSuite::ParamType;
TestMetaFactory() {}
TestFactoryBase* CreateTestFactory(ParamType parameter) override {
return new ParameterizedTestFactory<TestSuite>(parameter);
}
private:
TestMetaFactory(const TestMetaFactory&) = delete;
TestMetaFactory& operator=(const TestMetaFactory&) = delete;
};
class ParameterizedTestSuiteInfoBase {
public:
virtual ~ParameterizedTestSuiteInfoBase() {}
virtual const std::string& GetTestSuiteName() const = 0;
virtual TypeId GetTestSuiteTypeId() const = 0;
virtual void RegisterTests() = 0;
protected:
ParameterizedTestSuiteInfoBase() {}
private:
ParameterizedTestSuiteInfoBase(const ParameterizedTestSuiteInfoBase&) =
delete;
ParameterizedTestSuiteInfoBase& operator=(
const ParameterizedTestSuiteInfoBase&) = delete;
};
struct __attribute__((visibility("default"))) MarkAsIgnored {
explicit MarkAsIgnored(const char* test_suite);
};
__attribute__((visibility("default"))) void InsertSyntheticTestCase(const std::string& name,
CodeLocation location, bool has_test_p);
template <class TestSuite>
class ParameterizedTestSuiteInfo : public ParameterizedTestSuiteInfoBase {
public:
using ParamType = typename TestSuite::ParamType;
typedef ParamGenerator<ParamType>(GeneratorCreationFunc)();
using ParamNameGeneratorFunc = std::string(const TestParamInfo<ParamType>&);
explicit ParameterizedTestSuiteInfo(const char* name,
CodeLocation code_location)
: test_suite_name_(name), code_location_(code_location) {}
const std::string& GetTestSuiteName() const override {
return test_suite_name_;
}
TypeId GetTestSuiteTypeId() const override { return GetTypeId<TestSuite>(); }
void AddTestPattern(const char* test_suite_name, const char* test_base_name,
TestMetaFactoryBase<ParamType>* meta_factory,
CodeLocation code_location) {
tests_.push_back(std::shared_ptr<TestInfo>(new TestInfo(
test_suite_name, test_base_name, meta_factory, code_location)));
}
int AddTestSuiteInstantiation(const std::string& instantiation_name,
GeneratorCreationFunc* func,
ParamNameGeneratorFunc* name_func,
const char* file, int line) {
instantiations_.push_back(
InstantiationInfo(instantiation_name, func, name_func, file, line));
return 0;
}
void RegisterTests() override {
bool generated_instantiations = false;
for (typename TestInfoContainer::iterator test_it = tests_.begin();
test_it != tests_.end(); ++test_it) {
std::shared_ptr<TestInfo> test_info = *test_it;
for (typename InstantiationContainer::iterator gen_it =
instantiations_.begin();
gen_it != instantiations_.end(); ++gen_it) {
const std::string& instantiation_name = gen_it->name;
ParamGenerator<ParamType> generator((*gen_it->generator)());
ParamNameGeneratorFunc* name_func = gen_it->name_func;
const char* file = gen_it->file;
int line = gen_it->line;
std::string test_suite_name;
if (!instantiation_name.empty())
test_suite_name = instantiation_name + "/";
test_suite_name += test_info->test_suite_base_name;
size_t i = 0;
std::set<std::string> test_param_names;
for (typename ParamGenerator<ParamType>::iterator param_it =
generator.begin();
param_it != generator.end(); ++param_it, ++i) {
generated_instantiations = true;
Message test_name_stream;
std::string param_name =
name_func(TestParamInfo<ParamType>(*param_it, i));
switch (0) case 0: default: if (::testing::internal::IsTrue(IsValidParamName(param_name))) ; else ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-param-util.h", 585) .GetStream() << "Condition " "IsValidParamName(param_name)" " failed. "
<< "Parameterized test name '" << param_name
<< "' is invalid, in " << file << " line " << line << std::endl;
switch (0) case 0: default: if (::testing::internal::IsTrue(test_param_names.count(param_name) == 0)) ; else ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-param-util.h", 589) .GetStream() << "Condition " "test_param_names.count(param_name) == 0" " failed. "
<< "Duplicate parameterized test name '" << param_name << "', in "
<< file << " line " << line << std::endl;
test_param_names.insert(param_name);
if (!test_info->test_base_name.empty()) {
test_name_stream << test_info->test_base_name << "/";
}
test_name_stream << param_name;
MakeAndRegisterTestInfo(
test_suite_name.c_str(), test_name_stream.GetString().c_str(),
nullptr,
PrintToString(*param_it).c_str(), test_info->code_location,
GetTestSuiteTypeId(),
SuiteApiResolver<TestSuite>::GetSetUpCaseOrSuite(file, line),
SuiteApiResolver<TestSuite>::GetTearDownCaseOrSuite(file, line),
test_info->test_meta_factory->CreateTestFactory(*param_it));
}
}
}
if (!generated_instantiations) {
InsertSyntheticTestCase(GetTestSuiteName(), code_location_,
!tests_.empty());
}
}
private:
struct TestInfo {
TestInfo(const char* a_test_suite_base_name, const char* a_test_base_name,
TestMetaFactoryBase<ParamType>* a_test_meta_factory,
CodeLocation a_code_location)
: test_suite_base_name(a_test_suite_base_name),
test_base_name(a_test_base_name),
test_meta_factory(a_test_meta_factory),
code_location(a_code_location) {}
const std::string test_suite_base_name;
const std::string test_base_name;
const std::unique_ptr<TestMetaFactoryBase<ParamType>> test_meta_factory;
const CodeLocation code_location;
};
using TestInfoContainer = ::std::vector<std::shared_ptr<TestInfo>>;
struct InstantiationInfo {
InstantiationInfo(const std::string& name_in,
GeneratorCreationFunc* generator_in,
ParamNameGeneratorFunc* name_func_in, const char* file_in,
int line_in)
: name(name_in),
generator(generator_in),
name_func(name_func_in),
file(file_in),
line(line_in) {}
std::string name;
GeneratorCreationFunc* generator;
ParamNameGeneratorFunc* name_func;
const char* file;
int line;
};
typedef ::std::vector<InstantiationInfo> InstantiationContainer;
static bool IsValidParamName(const std::string& name) {
if (name.empty()) return false;
for (std::string::size_type index = 0; index < name.size(); ++index) {
if (!IsAlNum(name[index]) && name[index] != '_') return false;
}
return true;
}
const std::string test_suite_name_;
CodeLocation code_location_;
TestInfoContainer tests_;
InstantiationContainer instantiations_;
ParameterizedTestSuiteInfo(const ParameterizedTestSuiteInfo&) = delete;
ParameterizedTestSuiteInfo& operator=(const ParameterizedTestSuiteInfo&) =
delete;
};
template <class TestCase>
using ParameterizedTestCaseInfo = ParameterizedTestSuiteInfo<TestCase>;
class ParameterizedTestSuiteRegistry {
public:
ParameterizedTestSuiteRegistry() {}
~ParameterizedTestSuiteRegistry() {
for (auto& test_suite_info : test_suite_infos_) {
delete test_suite_info;
}
}
template <class TestSuite>
ParameterizedTestSuiteInfo<TestSuite>* GetTestSuitePatternHolder(
const char* test_suite_name, CodeLocation code_location) {
ParameterizedTestSuiteInfo<TestSuite>* typed_test_info = nullptr;
for (auto& test_suite_info : test_suite_infos_) {
if (test_suite_info->GetTestSuiteName() == test_suite_name) {
if (test_suite_info->GetTestSuiteTypeId() != GetTypeId<TestSuite>()) {
ReportInvalidTestSuiteType(test_suite_name, code_location);
posix::Abort();
} else {
typed_test_info = CheckedDowncastToActualType<
ParameterizedTestSuiteInfo<TestSuite>>(test_suite_info);
}
break;
}
}
if (typed_test_info == nullptr) {
typed_test_info = new ParameterizedTestSuiteInfo<TestSuite>(
test_suite_name, code_location);
test_suite_infos_.push_back(typed_test_info);
}
return typed_test_info;
}
void RegisterTests() {
for (auto& test_suite_info : test_suite_infos_) {
test_suite_info->RegisterTests();
}
}
template <class TestCase>
ParameterizedTestCaseInfo<TestCase>* GetTestCasePatternHolder(
const char* test_case_name, CodeLocation code_location) {
return GetTestSuitePatternHolder<TestCase>(test_case_name, code_location);
}
private:
using TestSuiteInfoContainer = ::std::vector<ParameterizedTestSuiteInfoBase*>;
TestSuiteInfoContainer test_suite_infos_;
ParameterizedTestSuiteRegistry(const ParameterizedTestSuiteRegistry&) =
delete;
ParameterizedTestSuiteRegistry& operator=(
const ParameterizedTestSuiteRegistry&) = delete;
};
class TypeParameterizedTestSuiteRegistry {
public:
void RegisterTestSuite(const char* test_suite_name,
CodeLocation code_location);
void RegisterInstantiation(const char* test_suite_name);
void CheckForInstantiations();
private:
struct TypeParameterizedTestSuiteInfo {
explicit TypeParameterizedTestSuiteInfo(CodeLocation c)
: code_location(c), instantiated(false) {}
CodeLocation code_location;
bool instantiated;
};
std::map<std::string, TypeParameterizedTestSuiteInfo> suites_;
};
}
template <class Container>
internal::ParamGenerator<typename Container::value_type> ValuesIn(
const Container& container);
namespace internal {
template <typename... Ts>
class ValueArray {
public:
explicit ValueArray(Ts... v) : v_(FlatTupleConstructTag{}, std::move(v)...) {}
template <typename T>
operator ParamGenerator<T>() const {
return ValuesIn(MakeVector<T>(MakeIndexSequence<sizeof...(Ts)>()));
}
private:
template <typename T, size_t... I>
std::vector<T> MakeVector(IndexSequence<I...>) const {
return std::vector<T>{static_cast<T>(v_.template Get<I>())...};
}
FlatTuple<Ts...> v_;
};
template <typename... T>
class CartesianProductGenerator
: public ParamGeneratorInterface<::std::tuple<T...>> {
public:
typedef ::std::tuple<T...> ParamType;
CartesianProductGenerator(const std::tuple<ParamGenerator<T>...>& g)
: generators_(g) {}
~CartesianProductGenerator() override {}
ParamIteratorInterface<ParamType>* Begin() const override {
return new Iterator(this, generators_, false);
}
ParamIteratorInterface<ParamType>* End() const override {
return new Iterator(this, generators_, true);
}
private:
template <class I>
class IteratorImpl;
template <size_t... I>
class IteratorImpl<IndexSequence<I...>>
: public ParamIteratorInterface<ParamType> {
public:
IteratorImpl(const ParamGeneratorInterface<ParamType>* base,
const std::tuple<ParamGenerator<T>...>& generators,
bool is_end)
: base_(base),
begin_(std::get<I>(generators).begin()...),
end_(std::get<I>(generators).end()...),
current_(is_end ? end_ : begin_) {
ComputeCurrentValue();
}
~IteratorImpl() override {}
const ParamGeneratorInterface<ParamType>* BaseGenerator() const override {
return base_;
}
void Advance() override {
(static_cast<void> (0));
++std::get<sizeof...(T) - 1>(current_);
AdvanceIfEnd<sizeof...(T) - 1>();
ComputeCurrentValue();
}
ParamIteratorInterface<ParamType>* Clone() const override {
return new IteratorImpl(*this);
}
const ParamType* Current() const override { return current_value_.get(); }
bool Equals(const ParamIteratorInterface<ParamType>& other) const override {
switch (0) case 0: default: if (::testing::internal::IsTrue(BaseGenerator() == other.BaseGenerator())) ; else ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-param-util.h", 882) .GetStream() << "Condition " "BaseGenerator() == other.BaseGenerator()" " failed. "
<< "The program attempted to compare iterators "
<< "from different generators." << std::endl;
const IteratorImpl* typed_other =
CheckedDowncastToActualType<const IteratorImpl>(&other);
if (AtEnd() && typed_other->AtEnd()) return true;
bool same = true;
bool dummy[] = {
(same = same && std::get<I>(current_) ==
std::get<I>(typed_other->current_))...};
(void)dummy;
return same;
}
private:
template <size_t ThisI>
void AdvanceIfEnd() {
if (std::get<ThisI>(current_) != std::get<ThisI>(end_)) return;
bool last = ThisI == 0;
if (last) {
return;
}
constexpr size_t NextI = ThisI - (ThisI != 0);
std::get<ThisI>(current_) = std::get<ThisI>(begin_);
++std::get<NextI>(current_);
AdvanceIfEnd<NextI>();
}
void ComputeCurrentValue() {
if (!AtEnd())
current_value_ = std::make_shared<ParamType>(*std::get<I>(current_)...);
}
bool AtEnd() const {
bool at_end = false;
bool dummy[] = {
(at_end = at_end || std::get<I>(current_) == std::get<I>(end_))...};
(void)dummy;
return at_end;
}
const ParamGeneratorInterface<ParamType>* const base_;
std::tuple<typename ParamGenerator<T>::iterator...> begin_;
std::tuple<typename ParamGenerator<T>::iterator...> end_;
std::tuple<typename ParamGenerator<T>::iterator...> current_;
std::shared_ptr<ParamType> current_value_;
};
using Iterator = IteratorImpl<typename MakeIndexSequence<sizeof...(T)>::type>;
std::tuple<ParamGenerator<T>...> generators_;
};
template <class... Gen>
class CartesianProductHolder {
public:
CartesianProductHolder(const Gen&... g) : generators_(g...) {}
template <typename... T>
operator ParamGenerator<::std::tuple<T...>>() const {
return ParamGenerator<::std::tuple<T...>>(
new CartesianProductGenerator<T...>(generators_));
}
private:
std::tuple<Gen...> generators_;
};
template <typename From, typename To>
class ParamGeneratorConverter : public ParamGeneratorInterface<To> {
public:
ParamGeneratorConverter(ParamGenerator<From> gen)
: generator_(std::move(gen)) {}
ParamIteratorInterface<To>* Begin() const override {
return new Iterator(this, generator_.begin(), generator_.end());
}
ParamIteratorInterface<To>* End() const override {
return new Iterator(this, generator_.end(), generator_.end());
}
private:
class Iterator : public ParamIteratorInterface<To> {
public:
Iterator(const ParamGeneratorInterface<To>* base, ParamIterator<From> it,
ParamIterator<From> end)
: base_(base), it_(it), end_(end) {
if (it_ != end_) value_ = std::make_shared<To>(static_cast<To>(*it_));
}
~Iterator() override {}
const ParamGeneratorInterface<To>* BaseGenerator() const override {
return base_;
}
void Advance() override {
++it_;
if (it_ != end_) value_ = std::make_shared<To>(static_cast<To>(*it_));
}
ParamIteratorInterface<To>* Clone() const override {
return new Iterator(*this);
}
const To* Current() const override { return value_.get(); }
bool Equals(const ParamIteratorInterface<To>& other) const override {
switch (0) case 0: default: if (::testing::internal::IsTrue(BaseGenerator() == other.BaseGenerator())) ; else ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/internal/gtest-param-util.h", 993) .GetStream() << "Condition " "BaseGenerator() == other.BaseGenerator()" " failed. "
<< "The program attempted to compare iterators "
<< "from different generators." << std::endl;
const ParamIterator<From> other_it =
CheckedDowncastToActualType<const Iterator>(&other)->it_;
return it_ == other_it;
}
private:
Iterator(const Iterator& other) = default;
const ParamGeneratorInterface<To>* const base_;
ParamIterator<From> it_;
ParamIterator<From> end_;
std::shared_ptr<To> value_;
};
ParamGenerator<From> generator_;
};
template <class Gen>
class ParamConverterGenerator {
public:
ParamConverterGenerator(ParamGenerator<Gen> g)
: generator_(std::move(g)) {}
template <typename T>
operator ParamGenerator<T>() const {
return ParamGenerator<T>(new ParamGeneratorConverter<Gen, T>(generator_));
}
private:
ParamGenerator<Gen> generator_;
};
}
}
namespace testing {
template <typename T, typename IncrementT>
internal::ParamGenerator<T> Range(T start, T end, IncrementT step) {
return internal::ParamGenerator<T>(
new internal::RangeGenerator<T, IncrementT>(start, end, step));
}
template <typename T>
internal::ParamGenerator<T> Range(T start, T end) {
return Range(start, end, 1);
}
template <typename ForwardIterator>
internal::ParamGenerator<
typename std::iterator_traits<ForwardIterator>::value_type>
ValuesIn(ForwardIterator begin, ForwardIterator end) {
typedef typename std::iterator_traits<ForwardIterator>::value_type ParamType;
return internal::ParamGenerator<ParamType>(
new internal::ValuesInIteratorRangeGenerator<ParamType>(begin, end));
}
template <typename T, size_t N>
internal::ParamGenerator<T> ValuesIn(const T (&array)[N]) {
return ValuesIn(array, array + N);
}
template <class Container>
internal::ParamGenerator<typename Container::value_type> ValuesIn(
const Container& container) {
return ValuesIn(container.begin(), container.end());
}
template <typename... T>
internal::ValueArray<T...> Values(T... v) {
return internal::ValueArray<T...>(std::move(v)...);
}
inline internal::ParamGenerator<bool> Bool() { return Values(false, true); }
template <typename... Generator>
internal::CartesianProductHolder<Generator...> Combine(const Generator&... g) {
return internal::CartesianProductHolder<Generator...>(g...);
}
template <typename T>
internal::ParamConverterGenerator<T> ConvertGenerator(
internal::ParamGenerator<T> gen) {
return internal::ParamConverterGenerator<T>(gen);
}
}
namespace testing {
template <typename Pred, typename T1>
AssertionResult AssertPred1Helper(const char* pred_text, const char* e1,
Pred pred, const T1& v1) {
if (pred(v1)) return AssertionSuccess();
return AssertionFailure()
<< pred_text << "(" << e1 << ") evaluates to false, where"
<< "\n"
<< e1 << " evaluates to " << ::testing::PrintToString(v1);
}
template <typename Pred, typename T1, typename T2>
AssertionResult AssertPred2Helper(const char* pred_text, const char* e1,
const char* e2, Pred pred, const T1& v1,
const T2& v2) {
if (pred(v1, v2)) return AssertionSuccess();
return AssertionFailure()
<< pred_text << "(" << e1 << ", " << e2
<< ") evaluates to false, where"
<< "\n"
<< e1 << " evaluates to " << ::testing::PrintToString(v1) << "\n"
<< e2 << " evaluates to " << ::testing::PrintToString(v2);
}
template <typename Pred, typename T1, typename T2, typename T3>
AssertionResult AssertPred3Helper(const char* pred_text, const char* e1,
const char* e2, const char* e3, Pred pred,
const T1& v1, const T2& v2, const T3& v3) {
if (pred(v1, v2, v3)) return AssertionSuccess();
return AssertionFailure()
<< pred_text << "(" << e1 << ", " << e2 << ", " << e3
<< ") evaluates to false, where"
<< "\n"
<< e1 << " evaluates to " << ::testing::PrintToString(v1) << "\n"
<< e2 << " evaluates to " << ::testing::PrintToString(v2) << "\n"
<< e3 << " evaluates to " << ::testing::PrintToString(v3);
}
template <typename Pred, typename T1, typename T2, typename T3, typename T4>
AssertionResult AssertPred4Helper(const char* pred_text, const char* e1,
const char* e2, const char* e3,
const char* e4, Pred pred, const T1& v1,
const T2& v2, const T3& v3, const T4& v4) {
if (pred(v1, v2, v3, v4)) return AssertionSuccess();
return AssertionFailure()
<< pred_text << "(" << e1 << ", " << e2 << ", " << e3 << ", " << e4
<< ") evaluates to false, where"
<< "\n"
<< e1 << " evaluates to " << ::testing::PrintToString(v1) << "\n"
<< e2 << " evaluates to " << ::testing::PrintToString(v2) << "\n"
<< e3 << " evaluates to " << ::testing::PrintToString(v3) << "\n"
<< e4 << " evaluates to " << ::testing::PrintToString(v4);
}
template <typename Pred, typename T1, typename T2, typename T3, typename T4,
typename T5>
AssertionResult AssertPred5Helper(const char* pred_text, const char* e1,
const char* e2, const char* e3,
const char* e4, const char* e5, Pred pred,
const T1& v1, const T2& v2, const T3& v3,
const T4& v4, const T5& v5) {
if (pred(v1, v2, v3, v4, v5)) return AssertionSuccess();
return AssertionFailure()
<< pred_text << "(" << e1 << ", " << e2 << ", " << e3 << ", " << e4
<< ", " << e5 << ") evaluates to false, where"
<< "\n"
<< e1 << " evaluates to " << ::testing::PrintToString(v1) << "\n"
<< e2 << " evaluates to " << ::testing::PrintToString(v2) << "\n"
<< e3 << " evaluates to " << ::testing::PrintToString(v3) << "\n"
<< e4 << " evaluates to " << ::testing::PrintToString(v4) << "\n"
<< e5 << " evaluates to " << ::testing::PrintToString(v5);
}
}
namespace testing { __attribute__((visibility("default"))) extern bool FLAGS_gtest_also_run_disabled_tests; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern bool FLAGS_gtest_break_on_failure; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern bool FLAGS_gtest_catch_exceptions; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern ::std::string FLAGS_gtest_color; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern bool FLAGS_gtest_fail_fast; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern ::std::string FLAGS_gtest_filter; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern bool FLAGS_gtest_install_failure_signal_handler; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern bool FLAGS_gtest_list_tests; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern ::std::string FLAGS_gtest_output; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern bool FLAGS_gtest_brief; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern bool FLAGS_gtest_print_time; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern bool FLAGS_gtest_print_utf8; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern std::int32_t FLAGS_gtest_random_seed; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern std::int32_t FLAGS_gtest_repeat; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern bool FLAGS_gtest_recreate_environments_when_repeating; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern bool FLAGS_gtest_show_internal_stack_frames; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern bool FLAGS_gtest_shuffle; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern std::int32_t FLAGS_gtest_stack_trace_depth; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern bool FLAGS_gtest_throw_on_failure; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern ::std::string FLAGS_gtest_stream_result_to; } static_assert(true, "no-op to require trailing semicolon");
namespace testing { __attribute__((visibility("default"))) extern ::std::string FLAGS_gtest_flagfile; } static_assert(true, "no-op to require trailing semicolon");
namespace testing {
const int kMaxStackTraceDepth = 100;
namespace internal {
class AssertHelper;
class DefaultGlobalTestPartResultReporter;
class ExecDeathTest;
class NoExecDeathTest;
class FinalSuccessChecker;
class GTestFlagSaver;
class StreamingListenerTest;
class TestResultAccessor;
class TestEventListenersAccessor;
class TestEventRepeater;
class UnitTestRecordPropertyTestHelper;
class WindowsDeathTest;
class FuchsiaDeathTest;
class UnitTestImpl* GetUnitTestImpl();
void ReportFailureInUnknownLocation(TestPartResult::Type result_type,
const std::string& message);
std::set<std::string>* GetIgnoredParameterizedTestSuites();
class GTestNonCopyable {
public:
GTestNonCopyable() = default;
GTestNonCopyable(const GTestNonCopyable &) = delete;
GTestNonCopyable &operator=(const GTestNonCopyable &) = delete;
~GTestNonCopyable() = default;
};
}
class Test;
class TestSuite;
using TestCase = TestSuite;
class TestInfo;
class UnitTest;
class __attribute__((visibility("default"))) Test {
public:
friend class TestInfo;
virtual ~Test();
static void SetUpTestSuite() {}
static void TearDownTestSuite() {}
static void TearDownTestCase() {}
static void SetUpTestCase() {}
static bool HasFatalFailure();
static bool HasNonfatalFailure();
static bool IsSkipped();
static bool HasFailure() { return HasFatalFailure() || HasNonfatalFailure(); }
static void RecordProperty(const std::string& key, const std::string& value);
static void RecordProperty(const std::string& key, int64_t value);
protected:
Test();
virtual void SetUp();
virtual void TearDown();
private:
static bool HasSameFixtureClass();
virtual void TestBody() = 0;
void Run();
void DeleteSelf_() { delete this; }
const std::unique_ptr< ::testing::internal::GTestFlagSaver> gtest_flag_saver_;
struct Setup_should_be_spelled_SetUp {};
virtual Setup_should_be_spelled_SetUp* Setup() { return nullptr; }
Test(const Test&) = delete;
Test& operator=(const Test&) = delete;
};
typedef internal::TimeInMillis TimeInMillis;
class TestProperty {
public:
TestProperty(const std::string& a_key, const std::string& a_value)
: key_(a_key), value_(a_value) {}
const char* key() const { return key_.c_str(); }
const char* value() const { return value_.c_str(); }
void SetValue(const std::string& new_value) { value_ = new_value; }
private:
std::string key_;
std::string value_;
};
class __attribute__((visibility("default"))) TestResult {
public:
TestResult();
~TestResult();
int total_part_count() const;
int test_property_count() const;
bool Passed() const { return !Skipped() && !Failed(); }
bool Skipped() const;
bool Failed() const;
bool HasFatalFailure() const;
bool HasNonfatalFailure() const;
TimeInMillis elapsed_time() const { return elapsed_time_; }
TimeInMillis start_timestamp() const { return start_timestamp_; }
const TestPartResult& GetTestPartResult(int i) const;
const TestProperty& GetTestProperty(int i) const;
private:
friend class TestInfo;
friend class TestSuite;
friend class UnitTest;
friend class internal::DefaultGlobalTestPartResultReporter;
friend class internal::ExecDeathTest;
friend class internal::TestResultAccessor;
friend class internal::UnitTestImpl;
friend class internal::WindowsDeathTest;
friend class internal::FuchsiaDeathTest;
const std::vector<TestPartResult>& test_part_results() const {
return test_part_results_;
}
const std::vector<TestProperty>& test_properties() const {
return test_properties_;
}
void set_start_timestamp(TimeInMillis start) { start_timestamp_ = start; }
void set_elapsed_time(TimeInMillis elapsed) { elapsed_time_ = elapsed; }
void RecordProperty(const std::string& xml_element,
const TestProperty& test_property);
static bool ValidateTestProperty(const std::string& xml_element,
const TestProperty& test_property);
void AddTestPartResult(const TestPartResult& test_part_result);
int death_test_count() const { return death_test_count_; }
int increment_death_test_count() { return ++death_test_count_; }
void ClearTestPartResults();
void Clear();
internal::Mutex test_properties_mutex_;
std::vector<TestPartResult> test_part_results_;
std::vector<TestProperty> test_properties_;
int death_test_count_;
TimeInMillis start_timestamp_;
TimeInMillis elapsed_time_;
TestResult(const TestResult&) = delete;
TestResult& operator=(const TestResult&) = delete;
};
class __attribute__((visibility("default"))) TestInfo {
public:
~TestInfo();
const char* test_suite_name() const { return test_suite_name_.c_str(); }
const char* test_case_name() const { return test_suite_name(); }
const char* name() const { return name_.c_str(); }
const char* type_param() const {
if (type_param_.get() != nullptr) return type_param_->c_str();
return nullptr;
}
const char* value_param() const {
if (value_param_.get() != nullptr) return value_param_->c_str();
return nullptr;
}
const char* file() const { return location_.file.c_str(); }
int line() const { return location_.line; }
bool is_in_another_shard() const { return is_in_another_shard_; }
bool should_run() const { return should_run_; }
bool is_reportable() const {
return matches_filter_ && !is_in_another_shard_;
}
const TestResult* result() const { return &result_; }
private:
friend class internal::DefaultDeathTestFactory;
friend class Test;
friend class TestSuite;
friend class internal::UnitTestImpl;
friend class internal::StreamingListenerTest;
friend TestInfo* internal::MakeAndRegisterTestInfo(
const char* test_suite_name, const char* name, const char* type_param,
const char* value_param, internal::CodeLocation code_location,
internal::TypeId fixture_class_id, internal::SetUpTestSuiteFunc set_up_tc,
internal::TearDownTestSuiteFunc tear_down_tc,
internal::TestFactoryBase* factory);
TestInfo(const std::string& test_suite_name, const std::string& name,
const char* a_type_param,
const char* a_value_param,
internal::CodeLocation a_code_location,
internal::TypeId fixture_class_id,
internal::TestFactoryBase* factory);
int increment_death_test_count() {
return result_.increment_death_test_count();
}
void Run();
void Skip();
static void ClearTestResult(TestInfo* test_info) {
test_info->result_.Clear();
}
const std::string test_suite_name_;
const std::string name_;
const std::unique_ptr<const ::std::string> type_param_;
const std::unique_ptr<const ::std::string> value_param_;
internal::CodeLocation location_;
const internal::TypeId fixture_class_id_;
bool should_run_;
bool is_disabled_;
bool matches_filter_;
bool is_in_another_shard_;
internal::TestFactoryBase* const factory_;
TestResult result_;
TestInfo(const TestInfo&) = delete;
TestInfo& operator=(const TestInfo&) = delete;
};
class __attribute__((visibility("default"))) TestSuite {
public:
TestSuite(const char* name, const char* a_type_param,
internal::SetUpTestSuiteFunc set_up_tc,
internal::TearDownTestSuiteFunc tear_down_tc);
virtual ~TestSuite();
const char* name() const { return name_.c_str(); }
const char* type_param() const {
if (type_param_.get() != nullptr) return type_param_->c_str();
return nullptr;
}
bool should_run() const { return should_run_; }
int successful_test_count() const;
int skipped_test_count() const;
int failed_test_count() const;
int reportable_disabled_test_count() const;
int disabled_test_count() const;
int reportable_test_count() const;
int test_to_run_count() const;
int total_test_count() const;
bool Passed() const { return !Failed(); }
bool Failed() const {
return failed_test_count() > 0 || ad_hoc_test_result().Failed();
}
TimeInMillis elapsed_time() const { return elapsed_time_; }
TimeInMillis start_timestamp() const { return start_timestamp_; }
const TestInfo* GetTestInfo(int i) const;
const TestResult& ad_hoc_test_result() const { return ad_hoc_test_result_; }
private:
friend class Test;
friend class internal::UnitTestImpl;
std::vector<TestInfo*>& test_info_list() { return test_info_list_; }
const std::vector<TestInfo*>& test_info_list() const {
return test_info_list_;
}
TestInfo* GetMutableTestInfo(int i);
void set_should_run(bool should) { should_run_ = should; }
void AddTestInfo(TestInfo* test_info);
void ClearResult();
static void ClearTestSuiteResult(TestSuite* test_suite) {
test_suite->ClearResult();
}
void Run();
void Skip();
void RunSetUpTestSuite() {
if (set_up_tc_ != nullptr) {
(*set_up_tc_)();
}
}
void RunTearDownTestSuite() {
if (tear_down_tc_ != nullptr) {
(*tear_down_tc_)();
}
}
static bool TestPassed(const TestInfo* test_info) {
return test_info->should_run() && test_info->result()->Passed();
}
static bool TestSkipped(const TestInfo* test_info) {
return test_info->should_run() && test_info->result()->Skipped();
}
static bool TestFailed(const TestInfo* test_info) {
return test_info->should_run() && test_info->result()->Failed();
}
static bool TestReportableDisabled(const TestInfo* test_info) {
return test_info->is_reportable() && test_info->is_disabled_;
}
static bool TestDisabled(const TestInfo* test_info) {
return test_info->is_disabled_;
}
static bool TestReportable(const TestInfo* test_info) {
return test_info->is_reportable();
}
static bool ShouldRunTest(const TestInfo* test_info) {
return test_info->should_run();
}
void ShuffleTests(internal::Random* random);
void UnshuffleTests();
std::string name_;
const std::unique_ptr<const ::std::string> type_param_;
std::vector<TestInfo*> test_info_list_;
std::vector<int> test_indices_;
internal::SetUpTestSuiteFunc set_up_tc_;
internal::TearDownTestSuiteFunc tear_down_tc_;
bool should_run_;
TimeInMillis start_timestamp_;
TimeInMillis elapsed_time_;
TestResult ad_hoc_test_result_;
TestSuite(const TestSuite&) = delete;
TestSuite& operator=(const TestSuite&) = delete;
};
class Environment {
public:
virtual ~Environment() {}
virtual void SetUp() {}
virtual void TearDown() {}
private:
struct Setup_should_be_spelled_SetUp {};
virtual Setup_should_be_spelled_SetUp* Setup() { return nullptr; }
};
class __attribute__((visibility("default"))) AssertionException
: public internal::GoogleTestFailureException {
public:
explicit AssertionException(const TestPartResult& result)
: GoogleTestFailureException(result) {}
};
class TestEventListener {
public:
virtual ~TestEventListener() {}
virtual void OnTestProgramStart(const UnitTest& unit_test) = 0;
virtual void OnTestIterationStart(const UnitTest& unit_test,
int iteration) = 0;
virtual void OnEnvironmentsSetUpStart(const UnitTest& unit_test) = 0;
virtual void OnEnvironmentsSetUpEnd(const UnitTest& unit_test) = 0;
virtual void OnTestSuiteStart(const TestSuite& ) {}
virtual void OnTestCaseStart(const TestCase& ) {}
virtual void OnTestStart(const TestInfo& test_info) = 0;
virtual void OnTestDisabled(const TestInfo& ) {}
virtual void OnTestPartResult(const TestPartResult& test_part_result) = 0;
virtual void OnTestEnd(const TestInfo& test_info) = 0;
virtual void OnTestSuiteEnd(const TestSuite& ) {}
virtual void OnTestCaseEnd(const TestCase& ) {}
virtual void OnEnvironmentsTearDownStart(const UnitTest& unit_test) = 0;
virtual void OnEnvironmentsTearDownEnd(const UnitTest& unit_test) = 0;
virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration) = 0;
virtual void OnTestProgramEnd(const UnitTest& unit_test) = 0;
};
class EmptyTestEventListener : public TestEventListener {
public:
void OnTestProgramStart(const UnitTest& ) override {}
void OnTestIterationStart(const UnitTest& ,
int ) override {}
void OnEnvironmentsSetUpStart(const UnitTest& ) override {}
void OnEnvironmentsSetUpEnd(const UnitTest& ) override {}
void OnTestSuiteStart(const TestSuite& ) override {}
void OnTestCaseStart(const TestCase& ) override {}
void OnTestStart(const TestInfo& ) override {}
void OnTestDisabled(const TestInfo& ) override {}
void OnTestPartResult(const TestPartResult& ) override {}
void OnTestEnd(const TestInfo& ) override {}
void OnTestSuiteEnd(const TestSuite& ) override {}
void OnTestCaseEnd(const TestCase& ) override {}
void OnEnvironmentsTearDownStart(const UnitTest& ) override {}
void OnEnvironmentsTearDownEnd(const UnitTest& ) override {}
void OnTestIterationEnd(const UnitTest& ,
int ) override {}
void OnTestProgramEnd(const UnitTest& ) override {}
};
class __attribute__((visibility("default"))) TestEventListeners {
public:
TestEventListeners();
~TestEventListeners();
void Append(TestEventListener* listener);
TestEventListener* Release(TestEventListener* listener);
TestEventListener* default_result_printer() const {
return default_result_printer_;
}
TestEventListener* default_xml_generator() const {
return default_xml_generator_;
}
private:
friend class TestSuite;
friend class TestInfo;
friend class internal::DefaultGlobalTestPartResultReporter;
friend class internal::NoExecDeathTest;
friend class internal::TestEventListenersAccessor;
friend class internal::UnitTestImpl;
TestEventListener* repeater();
void SetDefaultResultPrinter(TestEventListener* listener);
void SetDefaultXmlGenerator(TestEventListener* listener);
bool EventForwardingEnabled() const;
void SuppressEventForwarding();
internal::TestEventRepeater* repeater_;
TestEventListener* default_result_printer_;
TestEventListener* default_xml_generator_;
TestEventListeners(const TestEventListeners&) = delete;
TestEventListeners& operator=(const TestEventListeners&) = delete;
};
class __attribute__((visibility("default"))) UnitTest {
public:
static UnitTest* GetInstance();
int Run() __attribute__((warn_unused_result));
const char* original_working_dir() const;
const TestSuite* current_test_suite() const ;
const TestCase* current_test_case() const ;
const TestInfo* current_test_info() const ;
int random_seed() const;
internal::ParameterizedTestSuiteRegistry& parameterized_test_registry()
;
int successful_test_suite_count() const;
int failed_test_suite_count() const;
int total_test_suite_count() const;
int test_suite_to_run_count() const;
int successful_test_case_count() const;
int failed_test_case_count() const;
int total_test_case_count() const;
int test_case_to_run_count() const;
int successful_test_count() const;
int skipped_test_count() const;
int failed_test_count() const;
int reportable_disabled_test_count() const;
int disabled_test_count() const;
int reportable_test_count() const;
int total_test_count() const;
int test_to_run_count() const;
TimeInMillis start_timestamp() const;
TimeInMillis elapsed_time() const;
bool Passed() const;
bool Failed() const;
const TestSuite* GetTestSuite(int i) const;
const TestCase* GetTestCase(int i) const;
const TestResult& ad_hoc_test_result() const;
TestEventListeners& listeners();
private:
Environment* AddEnvironment(Environment* env);
void AddTestPartResult(TestPartResult::Type result_type,
const char* file_name, int line_number,
const std::string& message,
const std::string& os_stack_trace)
;
void RecordProperty(const std::string& key, const std::string& value);
TestSuite* GetMutableTestSuite(int i);
internal::UnitTestImpl* impl() { return impl_; }
const internal::UnitTestImpl* impl() const { return impl_; }
friend class ScopedTrace;
friend class Test;
friend class internal::AssertHelper;
friend class internal::StreamingListenerTest;
friend class internal::UnitTestRecordPropertyTestHelper;
friend Environment* AddGlobalTestEnvironment(Environment* env);
friend std::set<std::string>* internal::GetIgnoredParameterizedTestSuites();
friend internal::UnitTestImpl* internal::GetUnitTestImpl();
friend void internal::ReportFailureInUnknownLocation(
TestPartResult::Type result_type, const std::string& message);
UnitTest();
virtual ~UnitTest();
void PushGTestTrace(const internal::TraceInfo& trace)
;
void PopGTestTrace() ;
mutable internal::Mutex mutex_;
internal::UnitTestImpl* impl_;
UnitTest(const UnitTest&) = delete;
UnitTest& operator=(const UnitTest&) = delete;
};
inline Environment* AddGlobalTestEnvironment(Environment* env) {
return UnitTest::GetInstance()->AddEnvironment(env);
}
__attribute__((visibility("default"))) void InitGoogleTest(int* argc, char** argv);
__attribute__((visibility("default"))) void InitGoogleTest(int* argc, wchar_t** argv);
__attribute__((visibility("default"))) void InitGoogleTest();
namespace internal {
template <typename T1, typename T2>
AssertionResult CmpHelperEQFailure(const char* lhs_expression,
const char* rhs_expression, const T1& lhs,
const T2& rhs) {
return EqFailure(lhs_expression, rhs_expression,
FormatForComparisonFailureMessage(lhs, rhs),
FormatForComparisonFailureMessage(rhs, lhs), false);
}
struct faketype {};
inline bool operator==(faketype, faketype) { return true; }
inline bool operator!=(faketype, faketype) { return false; }
template <typename T1, typename T2>
AssertionResult CmpHelperEQ(const char* lhs_expression,
const char* rhs_expression, const T1& lhs,
const T2& rhs) {
if (lhs == rhs) {
return AssertionSuccess();
}
return CmpHelperEQFailure(lhs_expression, rhs_expression, lhs, rhs);
}
class EqHelper {
public:
template <
typename T1, typename T2,
typename std::enable_if<!std::is_integral<T1>::value ||
!std::is_pointer<T2>::value>::type* = nullptr>
static AssertionResult Compare(const char* lhs_expression,
const char* rhs_expression, const T1& lhs,
const T2& rhs) {
return CmpHelperEQ(lhs_expression, rhs_expression, lhs, rhs);
}
static AssertionResult Compare(const char* lhs_expression,
const char* rhs_expression, BiggestInt lhs,
BiggestInt rhs) {
return CmpHelperEQ(lhs_expression, rhs_expression, lhs, rhs);
}
template <typename T>
static AssertionResult Compare(
const char* lhs_expression, const char* rhs_expression,
std::nullptr_t , T* rhs) {
return CmpHelperEQ(lhs_expression, rhs_expression, static_cast<T*>(nullptr),
rhs);
}
};
template <typename T1, typename T2>
AssertionResult CmpHelperOpFailure(const char* expr1, const char* expr2,
const T1& val1, const T2& val2,
const char* op) {
return AssertionFailure()
<< "Expected: (" << expr1 << ") " << op << " (" << expr2
<< "), actual: " << FormatForComparisonFailureMessage(val1, val2)
<< " vs " << FormatForComparisonFailureMessage(val2, val1);
}
template <typename T1, typename T2> AssertionResult CmpHelperNE(const char* expr1, const char* expr2, const T1& val1, const T2& val2) { if (val1 != val2) { return AssertionSuccess(); } else { return CmpHelperOpFailure(expr1, expr2, val1, val2, "!="); } }
template <typename T1, typename T2> AssertionResult CmpHelperLE(const char* expr1, const char* expr2, const T1& val1, const T2& val2) { if (val1 <= val2) { return AssertionSuccess(); } else { return CmpHelperOpFailure(expr1, expr2, val1, val2, "<="); } }
template <typename T1, typename T2> AssertionResult CmpHelperLT(const char* expr1, const char* expr2, const T1& val1, const T2& val2) { if (val1 < val2) { return AssertionSuccess(); } else { return CmpHelperOpFailure(expr1, expr2, val1, val2, "<"); } }
template <typename T1, typename T2> AssertionResult CmpHelperGE(const char* expr1, const char* expr2, const T1& val1, const T2& val2) { if (val1 >= val2) { return AssertionSuccess(); } else { return CmpHelperOpFailure(expr1, expr2, val1, val2, ">="); } }
template <typename T1, typename T2> AssertionResult CmpHelperGT(const char* expr1, const char* expr2, const T1& val1, const T2& val2) { if (val1 > val2) { return AssertionSuccess(); } else { return CmpHelperOpFailure(expr1, expr2, val1, val2, ">"); } }
__attribute__((visibility("default"))) AssertionResult CmpHelperSTREQ(const char* s1_expression,
const char* s2_expression,
const char* s1, const char* s2);
__attribute__((visibility("default"))) AssertionResult CmpHelperSTRCASEEQ(const char* s1_expression,
const char* s2_expression,
const char* s1, const char* s2);
__attribute__((visibility("default"))) AssertionResult CmpHelperSTRNE(const char* s1_expression,
const char* s2_expression,
const char* s1, const char* s2);
__attribute__((visibility("default"))) AssertionResult CmpHelperSTRCASENE(const char* s1_expression,
const char* s2_expression,
const char* s1, const char* s2);
__attribute__((visibility("default"))) AssertionResult CmpHelperSTREQ(const char* s1_expression,
const char* s2_expression,
const wchar_t* s1, const wchar_t* s2);
__attribute__((visibility("default"))) AssertionResult CmpHelperSTRNE(const char* s1_expression,
const char* s2_expression,
const wchar_t* s1, const wchar_t* s2);
}
__attribute__((visibility("default"))) AssertionResult IsSubstring(const char* needle_expr,
const char* haystack_expr,
const char* needle,
const char* haystack);
__attribute__((visibility("default"))) AssertionResult IsSubstring(const char* needle_expr,
const char* haystack_expr,
const wchar_t* needle,
const wchar_t* haystack);
__attribute__((visibility("default"))) AssertionResult IsNotSubstring(const char* needle_expr,
const char* haystack_expr,
const char* needle,
const char* haystack);
__attribute__((visibility("default"))) AssertionResult IsNotSubstring(const char* needle_expr,
const char* haystack_expr,
const wchar_t* needle,
const wchar_t* haystack);
__attribute__((visibility("default"))) AssertionResult IsSubstring(const char* needle_expr,
const char* haystack_expr,
const ::std::string& needle,
const ::std::string& haystack);
__attribute__((visibility("default"))) AssertionResult IsNotSubstring(const char* needle_expr,
const char* haystack_expr,
const ::std::string& needle,
const ::std::string& haystack);
__attribute__((visibility("default"))) AssertionResult IsSubstring(const char* needle_expr,
const char* haystack_expr,
const ::std::wstring& needle,
const ::std::wstring& haystack);
__attribute__((visibility("default"))) AssertionResult IsNotSubstring(const char* needle_expr,
const char* haystack_expr,
const ::std::wstring& needle,
const ::std::wstring& haystack);
namespace internal {
template <typename RawType>
AssertionResult CmpHelperFloatingPointEQ(const char* lhs_expression,
const char* rhs_expression,
RawType lhs_value, RawType rhs_value) {
const FloatingPoint<RawType> lhs(lhs_value), rhs(rhs_value);
if (lhs.AlmostEquals(rhs)) {
return AssertionSuccess();
}
::std::stringstream lhs_ss;
lhs_ss << std::setprecision(std::numeric_limits<RawType>::digits10 + 2)
<< lhs_value;
::std::stringstream rhs_ss;
rhs_ss << std::setprecision(std::numeric_limits<RawType>::digits10 + 2)
<< rhs_value;
return EqFailure(lhs_expression, rhs_expression,
StringStreamToString(&lhs_ss), StringStreamToString(&rhs_ss),
false);
}
__attribute__((visibility("default"))) AssertionResult DoubleNearPredFormat(const char* expr1,
const char* expr2,
const char* abs_error_expr,
double val1, double val2,
double abs_error);
class __attribute__((visibility("default"))) AssertHelper {
public:
AssertHelper(TestPartResult::Type type, const char* file, int line,
const char* message);
~AssertHelper();
void operator=(const Message& message) const;
private:
struct AssertHelperData {
AssertHelperData(TestPartResult::Type t, const char* srcfile, int line_num,
const char* msg)
: type(t), file(srcfile), line(line_num), message(msg) {}
TestPartResult::Type const type;
const char* const file;
int const line;
std::string const message;
private:
AssertHelperData(const AssertHelperData&) = delete;
AssertHelperData& operator=(const AssertHelperData&) = delete;
};
AssertHelperData* const data_;
AssertHelper(const AssertHelper&) = delete;
AssertHelper& operator=(const AssertHelper&) = delete;
};
}
template <typename T>
class WithParamInterface {
public:
typedef T ParamType;
virtual ~WithParamInterface() {}
static const ParamType& GetParam() {
switch (0) case 0: default: if (::testing::internal::IsTrue(parameter_ != nullptr)) ; else ::testing::internal::GTestLog(::testing::internal::GTEST_FATAL, "/usr/include/gtest/gtest.h", 1677) .GetStream() << "Condition " "parameter_ != nullptr" " failed. "
<< "GetParam() can only be called inside a value-parameterized test "
<< "-- did you intend to write TEST_P instead of TEST_F?";
return *parameter_;
}
private:
static void SetParam(const ParamType* parameter) { parameter_ = parameter; }
static const ParamType* parameter_;
template <class TestClass>
friend class internal::ParameterizedTestFactory;
};
template <typename T>
const T* WithParamInterface<T>::parameter_ = nullptr;
template <typename T>
class TestWithParam : public Test, public WithParamInterface<T> {};
__attribute__((visibility("default"))) AssertionResult FloatLE(const char* expr1, const char* expr2,
float val1, float val2);
__attribute__((visibility("default"))) AssertionResult DoubleLE(const char* expr1, const char* expr2,
double val1, double val2);
class __attribute__((visibility("default"))) ScopedTrace {
public:
template <typename T>
ScopedTrace(const char* file, int line, const T& message) {
PushTrace(file, line, (Message() << message).GetString());
}
ScopedTrace(const char* file, int line, const char* message) {
PushTrace(file, line, message ? message : "(null)");
}
ScopedTrace(const char* file, int line, const std::string& message) {
PushTrace(file, line, message);
}
~ScopedTrace();
private:
void PushTrace(const char* file, int line, std::string message);
ScopedTrace(const ScopedTrace&) = delete;
ScopedTrace& operator=(const ScopedTrace&) = delete;
};
template <typename T1, typename T2>
constexpr bool StaticAssertTypeEq() noexcept {
static_assert(std::is_same<T1, T2>::value, "T1 and T2 are not the same type");
return true;
}
__attribute__((visibility("default"))) std::string TempDir();
__attribute__((visibility("default"))) std::string SrcDir();
template <int&... ExplicitParameterBarrier, typename Factory>
TestInfo* RegisterTest(const char* test_suite_name, const char* test_name,
const char* type_param, const char* value_param,
const char* file, int line, Factory factory) {
using TestT = typename std::remove_pointer<decltype(factory())>::type;
class FactoryImpl : public internal::TestFactoryBase {
public:
explicit FactoryImpl(Factory f) : factory_(std::move(f)) {}
Test* CreateTest() override { return factory_(); }
private:
Factory factory_;
};
return internal::MakeAndRegisterTestInfo(
test_suite_name, test_name, type_param, value_param,
internal::CodeLocation(file, line), internal::GetTypeId<TestT>(),
internal::SuiteApiResolver<TestT>::GetSetUpCaseOrSuite(file, line),
internal::SuiteApiResolver<TestT>::GetTearDownCaseOrSuite(file, line),
new FactoryImpl{std::move(factory)});
}
}
int RUN_ALL_TESTS() __attribute__((warn_unused_result));
inline int RUN_ALL_TESTS() { return ::testing::UnitTest::GetInstance()->Run(); }
namespace hwy {
class TestWithParamTarget : public testing::TestWithParam<int64_t> {
protected:
void SetUp() override { SetSupportedTargetsForTest(GetParam()); }
void TearDown() override {
switch (0) case 0: default: if (const ::testing::AssertionResult gtest_ar_ = ::testing::AssertionResult(
GetChosenTarget().IsInitialized()
)) ; else ::testing::internal::AssertHelper(::testing::TestPartResult::kNonFatalFailure, "/home/mathieu/Perso/highway/hwy/tests/hwy_gtest.h", 55, ::testing::internal::GetBoolAssertionFailureMessage( gtest_ar_,
"GetChosenTarget().IsInitialized()"
,
"false"
,
"true"
) .c_str()) = ::testing::Message()
<< "This hwy target parametric test doesn't use dynamic-dispatch and "
"doesn't need to be parametric.";
SetSupportedTargetsForTest(0);
}
};
static inline std::string TestParamTargetName(
const testing::TestParamInfo<int64_t>& info) {
return TargetName(info.param);
}
template <typename T>
class TestWithParamTargetAndT
: public ::testing::TestWithParam<std::tuple<int64_t, T>> {
public:
using HwyParamType = T;
protected:
void SetUp() override {
SetSupportedTargetsForTest(std::get<0>(
::testing::TestWithParam<std::tuple<int64_t, T>>::GetParam()));
}
void TearDown() override {
switch (0) case 0: default: if (const ::testing::AssertionResult gtest_ar_ = ::testing::AssertionResult(
GetChosenTarget().IsInitialized()
)) ; else ::testing::internal::AssertHelper(::testing::TestPartResult::kNonFatalFailure, "/home/mathieu/Perso/highway/hwy/tests/hwy_gtest.h", 104, ::testing::internal::GetBoolAssertionFailureMessage( gtest_ar_,
"GetChosenTarget().IsInitialized()"
,
"false"
,
"true"
) .c_str()) = ::testing::Message()
<< "This hwy target parametric test doesn't use dynamic-dispatch and "
"doesn't need to be parametric.";
SetSupportedTargetsForTest(0);
}
T GetParam() {
return std::get<1>(
::testing::TestWithParam<std::tuple<int64_t, T>>::GetParam());
}
};
template <typename T>
std::string TestParamTargetNameAndT(
const testing::TestParamInfo<std::tuple<int64_t, T>>& info) {
return std::string(TargetName(std::get<0>(info.param))) + "_" +
::testing::PrintToString(std::get<1>(info.param));
}
}
namespace hwy {
namespace detail {
struct TypeInfo {
size_t sizeof_t;
bool is_float;
bool is_signed;
};
template <typename T>
inline __attribute__((always_inline)) TypeInfo MakeTypeInfo() {
TypeInfo info;
info.sizeof_t = sizeof(T);
info.is_float = IsFloat<T>();
info.is_signed = IsSigned<T>();
return info;
}
void TypeName(const TypeInfo& info, size_t N, char* string100);
void ToString(const TypeInfo& info, const void* ptr,
char* string100);
void PrintArray(const TypeInfo& info, const char* caption,
const void* array_void, size_t N,
size_t lane_u = 0, size_t max_lanes = 7);
}
template <typename T>
__attribute__((noinline)) void PrintValue(T value) {
char str[100];
detail::ToString(hwy::detail::MakeTypeInfo<T>(), &value, str);
fprintf(
stderr
, "%s,", str);
}
template <typename T>
__attribute__((noinline)) void PrintArray(const T* value, size_t count) {
detail::PrintArray(hwy::detail::MakeTypeInfo<T>(), "", value, count, 0,
count);
}
}
namespace hwy {
constexpr size_t kTestMaxVectorSize = 64;
class RandomState {
public:
explicit RandomState(const uint64_t seed = 0x123456789ull) {
s0_ = SplitMix64(seed + 0x9E3779B97F4A7C15ull);
s1_ = SplitMix64(s0_);
}
inline __attribute__((always_inline)) uint64_t operator()() {
uint64_t s1 = s0_;
const uint64_t s0 = s1_;
const uint64_t bits = s1 + s0;
s0_ = s0;
s1 ^= s1 << 23;
s1 ^= s0 ^ (s1 >> 18) ^ (s0 >> 5);
s1_ = s1;
return bits;
}
private:
static uint64_t SplitMix64(uint64_t z) {
z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull;
z = (z ^ (z >> 27)) * 0x94D049BB133111EBull;
return z ^ (z >> 31);
}
uint64_t s0_;
uint64_t s1_;
};
static inline __attribute__((always_inline)) uint32_t Random32(RandomState* rng) {
return static_cast<uint32_t>((*rng)());
}
static inline __attribute__((always_inline)) uint64_t Random64(RandomState* rng) { return (*rng)(); }
bool BytesEqual(const void* p1, const void* p2,
const size_t size, size_t* pos = nullptr);
void AssertStringEqual(const char* expected, const char* actual,
const char* target_name, const char* filename, int line);
namespace detail {
template <typename T, typename TU = MakeUnsigned<T>>
TU ComputeUlpDelta(const T expected, const T actual) {
if (expected == actual) return 0;
if (std::isnan(expected) && std::isnan(actual)) return 0;
TU ux, uy;
CopySameSize(&expected, &ux);
CopySameSize(&actual, &uy);
const TU ulp = ((ux) > (uy) ? (ux) : (uy)) - ((ux) < (uy) ? (ux) : (uy));
return ulp;
}
bool IsEqual(const TypeInfo& info, const void* expected_ptr,
const void* actual_ptr);
__attribute__((noreturn)) void PrintMismatchAndAbort(
const TypeInfo& info, const void* expected_ptr, const void* actual_ptr,
const char* target_name, const char* filename, int line, size_t lane = 0,
size_t num_lanes = 1);
void AssertArrayEqual(const TypeInfo& info,
const void* expected_void,
const void* actual_void, size_t N,
const char* target_name,
const char* filename, int line);
}
template <typename T>
std::string TypeName(T , size_t N) {
char string100[100];
detail::TypeName(detail::MakeTypeInfo<T>(), N, string100);
return string100;
}
template <typename T>
inline __attribute__((always_inline)) bool IsEqual(const T expected, const T actual) {
const auto info = detail::MakeTypeInfo<T>();
return detail::IsEqual(info, &expected, &actual);
}
template <typename T>
inline __attribute__((always_inline)) void AssertEqual(const T expected, const T actual,
const char* target_name, const char* filename,
int line, size_t lane = 0) {
const auto info = detail::MakeTypeInfo<T>();
if (!detail::IsEqual(info, &expected, &actual)) {
detail::PrintMismatchAndAbort(info, &expected, &actual, target_name,
filename, line, lane);
}
}
template <typename T>
inline __attribute__((always_inline)) void AssertArrayEqual(const T* expected, const T* actual,
size_t count, const char* target_name,
const char* filename, int line) {
const auto info = hwy::detail::MakeTypeInfo<T>();
detail::AssertArrayEqual(info, expected, actual, count, target_name, filename,
line);
}
}
#pragma GCC push_options
#pragma GCC target "sse2"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSE2 {
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Print(const D d, const char* caption, V v, size_t lane_u = 0,
size_t max_lanes = 7) {
const size_t N = Lanes(d);
using T = TFromD<D>;
auto lanes = AllocateAligned<T>(N);
Store(v, d, lanes.get());
const auto info = hwy::detail::MakeTypeInfo<T>();
hwy::detail::PrintArray(info, caption, lanes.get(), N, lane_u, max_lanes);
}
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC push_options
#pragma GCC target "sse2"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSE2 {
template <class D, typename T = TFromD<D>>
inline __attribute__((always_inline)) void AssertVecEqual(D d, const T* expected, Vec<D> actual,
const char* filename, const int line) {
const size_t N = Lanes(d);
auto actual_lanes = AllocateAligned<T>(N);
Store(actual, d, actual_lanes.get());
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName((1LL << 14));
hwy::detail::AssertArrayEqual(info, expected, actual_lanes.get(), N,
target_name, filename, line);
}
template <class D, typename T = TFromD<D>>
inline __attribute__((always_inline)) void AssertVecEqual(D d, Vec<D> expected, Vec<D> actual,
const char* filename, int line) {
const size_t N = Lanes(d);
auto expected_lanes = AllocateAligned<T>(N);
auto actual_lanes = AllocateAligned<T>(N);
Store(expected, d, expected_lanes.get());
Store(actual, d, actual_lanes.get());
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName((1LL << 14));
hwy::detail::AssertArrayEqual(info, expected_lanes.get(), actual_lanes.get(),
N, target_name, filename, line);
}
template <class D>
__attribute__((noinline)) void AssertMaskEqual(D d, VecArg<Mask<D>> a, VecArg<Mask<D>> b,
const char* filename, int line) {
const Vec<D> va = VecFromMask(d, a);
const Vec<D> vb = VecFromMask(d, b);
AssertVecEqual(d, va, vb, filename, line);
const char* target_name = hwy::TargetName((1LL << 14));
AssertEqual(CountTrue(d, a), CountTrue(d, b), target_name, filename, line);
AssertEqual(AllTrue(d, a), AllTrue(d, b), target_name, filename, line);
AssertEqual(AllFalse(d, a), AllFalse(d, b), target_name, filename, line);
const size_t N = Lanes(d);
const Repartition<uint8_t, D> d8;
const size_t N8 = Lanes(d8);
auto bits_a = AllocateAligned<uint8_t>(((size_t{8}) > (N8) ? (size_t{8}) : (N8)));
auto bits_b = AllocateAligned<uint8_t>(size_t{((8) > (N8) ? (8) : (N8))});
memset(bits_a.get(), 0, N8);
memset(bits_b.get(), 0, N8);
const size_t num_bytes_a = StoreMaskBits(d, a, bits_a.get());
const size_t num_bytes_b = StoreMaskBits(d, b, bits_b.get());
AssertEqual(num_bytes_a, num_bytes_b, target_name, filename, line);
size_t i = 0;
for (; i < N / 8; ++i) {
if (bits_a[i] != bits_b[i]) {
fprintf(
stderr
, "Mismatch in byte %d: %d != %d\n", static_cast<int>(i),
bits_a[i], bits_b[i]);
Print(d8, "expect", Load(d8, bits_a.get()), 0, N8);
Print(d8, "actual", Load(d8, bits_b.get()), 0, N8);
hwy::Abort(filename, line, "Masks not equal");
}
}
const size_t remainder = N % 8;
if (remainder != 0) {
const int mask = (1 << remainder) - 1;
const int valid_a = bits_a[i] & mask;
const int valid_b = bits_b[i] & mask;
if (valid_a != valid_b) {
fprintf(
stderr
, "Mismatch in last byte %d: %d != %d\n",
static_cast<int>(i), valid_a, valid_b);
Print(d8, "expect", Load(d8, bits_a.get()), 0, N8);
Print(d8, "actual", Load(d8, bits_b.get()), 0, N8);
hwy::Abort(filename, line, "Masks not equal");
}
}
}
template <class D>
inline __attribute__((always_inline)) Mask<D> MaskTrue(const D d) {
return FirstN(d, Lanes(d));
}
template <class D>
inline __attribute__((always_inline)) Mask<D> MaskFalse(const D d) {
const RebindToSigned<D> di;
const Vec<decltype(di)> zero = Zero(di);
return RebindMask(d, Lt(zero, zero));
}
namespace detail {
template <typename T, size_t kMul, size_t kMinArg, class Test, int kPow2 = 0>
struct ForeachCappedR {
static void Do(size_t min_lanes, size_t max_lanes) {
const CappedTag<T, kMul * kMinArg, kPow2> d;
const size_t lanes = Lanes(d);
if (lanes < min_lanes) return;
if (lanes <= max_lanes) {
Test()(T(), d);
}
ForeachCappedR<T, kMul / 2, kMinArg, Test, kPow2>::Do(min_lanes, max_lanes);
}
};
template <typename T, size_t kMinArg, class Test, int kPow2>
struct ForeachCappedR<T, 0, kMinArg, Test, kPow2> {
static void Do(size_t, size_t) {}
};
}
template <class Test>
class ForMaxPow2 {
mutable bool called_ = false;
public:
~ForMaxPow2() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 291, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
detail::ForeachCappedR<T, (16 / sizeof(T)), 1, Test>::Do(
1, Lanes(ScalableTag<T>()));
}
};
template <class Test, int kPow2 = 1>
class ForExtendableVectors {
mutable bool called_ = false;
public:
~ForExtendableVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 324, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMaxCapped = (16 / sizeof(T));
const size_t max_lanes = Lanes(ScalableTag<T>()) >> kPow2;
(void)kMaxCapped;
(void)max_lanes;
constexpr size_t kMul = kMaxCapped >> kPow2;
constexpr size_t kMinArg = size_t{1} << kPow2;
detail::ForeachCappedR<T, kMul, kMinArg, Test, -kPow2>::Do(1, max_lanes);
}
};
template <class Test, int kPow2 = 1>
class ForShrinkableVectors {
mutable bool called_ = false;
public:
~ForShrinkableVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 358, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMinLanes = size_t{1} << kPow2;
constexpr size_t kMaxCapped = (16 / sizeof(T));
constexpr size_t max_lanes = kMaxCapped;
(void)kMinLanes;
(void)max_lanes;
(void)max_lanes;
detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kMinLanes, Test>::Do(
kMinLanes, max_lanes);
}
};
template <size_t kMinBits, class Test>
class ForGEVectors {
mutable bool called_ = false;
public:
~ForGEVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 394, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMaxCapped = (16 / sizeof(T));
constexpr size_t kMinLanes = kMinBits / 8 / sizeof(T);
constexpr size_t max_lanes = kMaxCapped;
(void)max_lanes;
detail::ForeachCappedR<T, (16 / sizeof(T)) / kMinLanes, kMinLanes, Test>::Do(
kMinLanes, max_lanes);
}
};
template <class Test>
using ForGE128Vectors = ForGEVectors<128, Test>;
template <class Test, int kPow2 = 1>
class ForPromoteVectors {
mutable bool called_ = false;
public:
~ForPromoteVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 436, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kFactor = size_t{1} << kPow2;
static_assert(kFactor >= 2 && kFactor * sizeof(T) <= sizeof(uint64_t), "");
constexpr size_t kMaxCapped = (16 / sizeof(T));
const size_t max_lanes = Lanes(ScalableTag<T>()) >> kPow2;
(void)kMaxCapped;
(void)max_lanes;
using DLargestFrom = CappedTag<T, (kMaxCapped >> kPow2) * kFactor, -kPow2>;
static_assert(DLargestFrom::kPrivateLanes <= (kMaxCapped >> kPow2),
"HWY_MAX_LANES_D(DLargestFrom) must be less than or equal to "
"(kMaxCapped >> kPow2)");
detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kFactor, Test, -kPow2>::Do(
1, max_lanes);
}
};
template <class Test, int kPow2 = 1>
class ForDemoteVectors {
mutable bool called_ = false;
public:
~ForDemoteVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 475, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMinLanes = size_t{1} << kPow2;
constexpr size_t kMaxCapped = (16 / sizeof(T));
constexpr size_t max_lanes = kMaxCapped;
(void)kMinLanes;
(void)max_lanes;
(void)max_lanes;
detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kMinLanes, Test>::Do(
kMinLanes, max_lanes);
}
};
template <class Test, int kPow2 = 1>
class ForHalfVectors {
mutable bool called_ = false;
public:
~ForHalfVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 512, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMinLanes = size_t{1} << kPow2;
constexpr size_t kMaxCapped = (16 / sizeof(T));
detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kMinLanes, Test>::Do(
kMinLanes, kMaxCapped);
}
};
template <class Test>
class ForPartialVectors {
mutable bool called_ = false;
public:
~ForPartialVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 545, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T t) const {
called_ = true;
ForExtendableVectors<Test, 0>()(t);
}
};
template <class Test>
using ForPartialFixedOrFullScalableVectors = ForPartialVectors<Test>;
template <class Func>
void ForSignedTypes(const Func& func) {
func(int8_t());
func(int16_t());
func(int32_t());
func(int64_t());
}
template <class Func>
void ForUnsignedTypes(const Func& func) {
func(uint8_t());
func(uint16_t());
func(uint32_t());
func(uint64_t());
}
template <class Func>
void ForIntegerTypes(const Func& func) {
ForSignedTypes(func);
ForUnsignedTypes(func);
}
template <class Func>
void ForFloatTypes(const Func& func) {
func(float());
func(double());
}
template <class Func>
void ForAllTypes(const Func& func) {
ForIntegerTypes(func);
ForFloatTypes(func);
}
template <class Func>
void ForUI8(const Func& func) {
func(uint8_t());
func(int8_t());
}
template <class Func>
void ForUI16(const Func& func) {
func(uint16_t());
func(int16_t());
}
template <class Func>
void ForUIF16(const Func& func) {
ForUI16(func);
func(float16_t());
}
template <class Func>
void ForUI32(const Func& func) {
func(uint32_t());
func(int32_t());
}
template <class Func>
void ForUIF32(const Func& func) {
ForUI32(func);
func(float());
}
template <class Func>
void ForUI64(const Func& func) {
func(uint64_t());
func(int64_t());
}
template <class Func>
void ForUIF64(const Func& func) {
ForUI64(func);
func(double());
}
template <class Func>
void ForUI3264(const Func& func) {
ForUI32(func);
ForUI64(func);
}
template <class Func>
void ForUIF3264(const Func& func) {
ForUIF32(func);
ForUIF64(func);
}
template <class Func>
void ForU163264(const Func& func) {
func(uint16_t());
func(uint32_t());
func(uint64_t());
}
template <class Func>
void ForUI163264(const Func& func) {
ForUI16(func);
ForUI3264(func);
}
template <class Func>
void ForUIF163264(const Func& func) {
ForUIF16(func);
ForUIF3264(func);
}
constexpr size_t AdjustedReps(size_t max_reps) {
return ((max_reps) > (2) ? (max_reps) : (2));
}
constexpr size_t AdjustedLog2Reps(size_t max_pow2) {
return max_pow2;
}
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC push_options
#pragma GCC target "sse2"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSE2 {
template <class Out, class In>
inline Out BitCast(const In& in) {
static_assert(sizeof(Out) == sizeof(In), "");
Out out;
CopyBytes<sizeof(out)>(&in, &out);
return out;
}
template <class T, class D>
__attribute__((noinline)) void TestMath(const char* name, T (*fx1)(T),
Vec<D> (*fxN)(D, VecArg<Vec<D>>), D d, T min, T max,
uint64_t max_error_ulp) {
using UintT = MakeUnsigned<T>;
const UintT min_bits = BitCast<UintT>(min);
const UintT max_bits = BitCast<UintT>(max);
int range_count = 1;
UintT ranges[2][2] = {{min_bits, max_bits}, {0, 0}};
if ((min < 0.0) && (max > 0.0)) {
ranges[0][0] = BitCast<UintT>(static_cast<T>(+0.0));
ranges[0][1] = max_bits;
ranges[1][0] = BitCast<UintT>(static_cast<T>(-0.0));
ranges[1][1] = min_bits;
range_count = 2;
}
uint64_t max_ulp = 0;
constexpr UintT kSamplesPerRange = static_cast<UintT>(AdjustedReps(4000));
for (int range_index = 0; range_index < range_count; ++range_index) {
const UintT start = ranges[range_index][0];
const UintT stop = ranges[range_index][1];
const UintT step = ((1) > (((stop - start) / kSamplesPerRange)) ? (1) : (((stop - start) / kSamplesPerRange)));
for (UintT value_bits = start; value_bits <= stop; value_bits += step) {
const T value = BitCast<T>(((((start) > (value_bits) ? (start) : (value_bits))) < (stop) ? (((start) > (value_bits) ? (start) : (value_bits))) : (stop)));
const T actual = GetLane(fxN(d, Set(d, value)));
const T expected = fx1(value);
const auto ulp = hwy::detail::ComputeUlpDelta(actual, expected);
max_ulp = ((max_ulp) > (ulp) ? (max_ulp) : (ulp));
if (ulp > max_error_ulp) {
fprintf(
stderr
, "KO %s: %s(%.17g) expected %.17g actual %.17g ulp %g max ulp %u\n",
hwy::TypeName(T(), Lanes(d)).c_str(), name, value, expected,
actual, static_cast<double>(ulp),
static_cast<uint32_t>(max_error_ulp));
if( value == 5.526720574244119e-20 && expected == 5.526720574244119e-20 && actual == 0) exit(42);
}
else {
fprintf(
stderr
, "OK %s: %s(%.17g) expected %.17g actual %.17g ulp %g max ulp %u\n",
hwy::TypeName(T(), Lanes(d)).c_str(), name, value, expected,
actual, static_cast<double>(ulp),
static_cast<uint32_t>(max_error_ulp));
}
}
}
fprintf(
stderr
, "%s: %s max_ulp %g\n", hwy::TypeName(T(), Lanes(d)).c_str(),
name, static_cast<double>(max_ulp));
}
const float kNearOneF = BitCast<float>(0x3F7FFFFF);
const double kNearOneD = BitCast<double>(0x3FEFFFFFFFFFFFFFULL);
struct TestLog1p { template <class T, class D> __attribute__((noinline)) void operator()(T, D d) { if (sizeof(T) == 4) { TestMath<T, D>("Log1p", std::log1p, CallLog1p, d, +0.0f, +1e37f, 3); } else { TestMath<T, D>("Log1p", std::log1p, CallLog1p, d, static_cast<T>(+0.0), static_cast<T>(+double(1.79769313486231570814527423731704357e+308L)
), 2); } } }; __attribute__((noinline)) void TestAllLog1p() { ForFloatTypes(ForPartialVectors<TestLog1p>()); }
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wuninitialized"
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#pragma GCC push_options
#pragma GCC target("mwait")
extern __inline void
__attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_monitor (void const * __P, unsigned int __E, unsigned int __H)
{
__builtin_ia32_monitor (__P, __E, __H);
}
extern __inline void
__attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mwait (unsigned int __E, unsigned int __H)
{
__builtin_ia32_mwait (__E, __H);
}
#pragma GCC pop_options
#pragma GCC push_options
#pragma GCC target("sse3")
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_addsub_ps (__m128 __X, __m128 __Y)
{
return (__m128) __builtin_ia32_addsubps ((__v4sf)__X, (__v4sf)__Y);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_hadd_ps (__m128 __X, __m128 __Y)
{
return (__m128) __builtin_ia32_haddps ((__v4sf)__X, (__v4sf)__Y);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_hsub_ps (__m128 __X, __m128 __Y)
{
return (__m128) __builtin_ia32_hsubps ((__v4sf)__X, (__v4sf)__Y);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_movehdup_ps (__m128 __X)
{
return (__m128) __builtin_ia32_movshdup ((__v4sf)__X);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_moveldup_ps (__m128 __X)
{
return (__m128) __builtin_ia32_movsldup ((__v4sf)__X);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_addsub_pd (__m128d __X, __m128d __Y)
{
return (__m128d) __builtin_ia32_addsubpd ((__v2df)__X, (__v2df)__Y);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_hadd_pd (__m128d __X, __m128d __Y)
{
return (__m128d) __builtin_ia32_haddpd ((__v2df)__X, (__v2df)__Y);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_hsub_pd (__m128d __X, __m128d __Y)
{
return (__m128d) __builtin_ia32_hsubpd ((__v2df)__X, (__v2df)__Y);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_loaddup_pd (double const *__P)
{
return _mm_load1_pd (__P);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_movedup_pd (__m128d __X)
{
return _mm_shuffle_pd (__X, __X, (((0) << 1) | (0)));
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_lddqu_si128 (__m128i const *__P)
{
return (__m128i) __builtin_ia32_lddqu ((char const *)__P);
}
#pragma GCC pop_options
#pragma GCC push_options
#pragma GCC target("ssse3")
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_hadd_epi16 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_phaddw128 ((__v8hi)__X, (__v8hi)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_hadd_epi32 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_phaddd128 ((__v4si)__X, (__v4si)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_hadds_epi16 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_phaddsw128 ((__v8hi)__X, (__v8hi)__Y);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_hadd_pi16 (__m64 __X, __m64 __Y)
{
return (__m64) __builtin_ia32_phaddw ((__v4hi)__X, (__v4hi)__Y);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_hadd_pi32 (__m64 __X, __m64 __Y)
{
return (__m64) __builtin_ia32_phaddd ((__v2si)__X, (__v2si)__Y);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_hadds_pi16 (__m64 __X, __m64 __Y)
{
return (__m64) __builtin_ia32_phaddsw ((__v4hi)__X, (__v4hi)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_hsub_epi16 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_phsubw128 ((__v8hi)__X, (__v8hi)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_hsub_epi32 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_phsubd128 ((__v4si)__X, (__v4si)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_hsubs_epi16 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_phsubsw128 ((__v8hi)__X, (__v8hi)__Y);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_hsub_pi16 (__m64 __X, __m64 __Y)
{
return (__m64) __builtin_ia32_phsubw ((__v4hi)__X, (__v4hi)__Y);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_hsub_pi32 (__m64 __X, __m64 __Y)
{
return (__m64) __builtin_ia32_phsubd ((__v2si)__X, (__v2si)__Y);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_hsubs_pi16 (__m64 __X, __m64 __Y)
{
return (__m64) __builtin_ia32_phsubsw ((__v4hi)__X, (__v4hi)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_maddubs_epi16 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_pmaddubsw128 ((__v16qi)__X, (__v16qi)__Y);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_maddubs_pi16 (__m64 __X, __m64 __Y)
{
return (__m64) __builtin_ia32_pmaddubsw ((__v8qi)__X, (__v8qi)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mulhrs_epi16 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_pmulhrsw128 ((__v8hi)__X, (__v8hi)__Y);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mulhrs_pi16 (__m64 __X, __m64 __Y)
{
return (__m64) __builtin_ia32_pmulhrsw ((__v4hi)__X, (__v4hi)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_shuffle_epi8 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_pshufb128 ((__v16qi)__X, (__v16qi)__Y);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_shuffle_pi8 (__m64 __X, __m64 __Y)
{
return (__m64) __builtin_ia32_pshufb ((__v8qi)__X, (__v8qi)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sign_epi8 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_psignb128 ((__v16qi)__X, (__v16qi)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sign_epi16 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_psignw128 ((__v8hi)__X, (__v8hi)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sign_epi32 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_psignd128 ((__v4si)__X, (__v4si)__Y);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sign_pi8 (__m64 __X, __m64 __Y)
{
return (__m64) __builtin_ia32_psignb ((__v8qi)__X, (__v8qi)__Y);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sign_pi16 (__m64 __X, __m64 __Y)
{
return (__m64) __builtin_ia32_psignw ((__v4hi)__X, (__v4hi)__Y);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_sign_pi32 (__m64 __X, __m64 __Y)
{
return (__m64) __builtin_ia32_psignd ((__v2si)__X, (__v2si)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_alignr_epi8(__m128i __X, __m128i __Y, const int __N)
{
return (__m128i) __builtin_ia32_palignr128 ((__v2di)__X,
(__v2di)__Y, __N * 8);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_alignr_pi8(__m64 __X, __m64 __Y, const int __N)
{
return (__m64) __builtin_ia32_palignr ((__v1di)__X,
(__v1di)__Y, __N * 8);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_abs_epi8 (__m128i __X)
{
return (__m128i) __builtin_ia32_pabsb128 ((__v16qi)__X);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_abs_epi16 (__m128i __X)
{
return (__m128i) __builtin_ia32_pabsw128 ((__v8hi)__X);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_abs_epi32 (__m128i __X)
{
return (__m128i) __builtin_ia32_pabsd128 ((__v4si)__X);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_abs_pi8 (__m64 __X)
{
return (__m64) __builtin_ia32_pabsb ((__v8qi)__X);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_abs_pi16 (__m64 __X)
{
return (__m64) __builtin_ia32_pabsw ((__v4hi)__X);
}
extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_abs_pi32 (__m64 __X)
{
return (__m64) __builtin_ia32_pabsd ((__v2si)__X);
}
#pragma GCC pop_options
#pragma GCC push_options
#pragma GCC target "sse2,ssse3"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSSE3 {
template <class V>
using VecArg = V;
namespace detail {
constexpr size_t ScaleByPower(size_t N, int pow2) {
return pow2 >= 0 ? (N << pow2) : (N >> (-pow2));
}
template <typename T>
inline __attribute__((always_inline)) void MaybeUnpoison(T* __restrict__ unaligned, size_t count) {
(void)unaligned;
(void)count;
}
}
template <typename Lane, size_t N, int kPow2>
struct Simd {
constexpr Simd() = default;
using T = Lane;
private:
static_assert(sizeof(Lane) <= 8, "Lanes are up to 64-bit");
static constexpr size_t kWhole = N & 0xFFFFF;
static constexpr int kFrac = static_cast<int>(N >> 20);
static_assert(kWhole <= 8 * 65536 && kFrac <= 3, "Out of range");
static_assert(kFrac == 0 || kWhole == 1, "If frac, whole must be 1");
static_assert((kWhole & (kWhole - 1)) == 0 && kWhole != 0, "Not 2^x");
static_assert(kPow2 >= -8, "Forgot kPow2 recursion terminator?");
public:
static constexpr size_t kPrivateLanes =
((size_t{1}) > (detail::ScaleByPower(kWhole, kPow2 - kFrac)) ? (size_t{1}) : (detail::ScaleByPower(kWhole, kPow2 - kFrac)));
constexpr size_t MaxLanes() const { return kPrivateLanes; }
constexpr size_t MaxBytes() const { return kPrivateLanes * sizeof(Lane); }
constexpr size_t MaxBlocks() const { return (MaxBytes() + 15) / 16; }
constexpr int Pow2() const { return kPow2; }
template <typename NewT>
static constexpr size_t RepartitionLanes() {
return (kPrivateLanes * sizeof(T) + sizeof(NewT) - 1) / sizeof(NewT);
}
template <typename NewT>
static constexpr int RebindPow2() {
return kPow2 +
((sizeof(NewT) >= sizeof(T))
? static_cast<int>(CeilLog2(sizeof(NewT) / sizeof(T)))
: -static_cast<int>(CeilLog2(sizeof(T) / sizeof(NewT))));
}
private:
template <int kNewPow2, size_t kNewMaxLanes>
static constexpr size_t WholeN() {
return detail::ScaleByPower(kNewMaxLanes, -kNewPow2);
}
template <int kNewPow2, size_t kNewMaxLanes>
static constexpr size_t FracN() {
static_assert(65536 <= (size_t{1} << 20), "Change bit shift");
return static_cast<size_t>(
1 + (((0) > (kNewPow2 - static_cast<int>(CeilLog2(kNewMaxLanes))) ? (0) : (kNewPow2 - static_cast<int>(CeilLog2(kNewMaxLanes))))
<< 20));
}
public:
template <int kNewPow2, size_t kNewMaxLanes>
static constexpr size_t NewN() {
return WholeN<kNewPow2, kNewMaxLanes>() == 0
? FracN<kNewPow2, kNewMaxLanes>()
: WholeN<kNewPow2, kNewMaxLanes>();
}
template <typename NewT>
using Rebind =
Simd<NewT, NewN<RebindPow2<NewT>(), kPrivateLanes>(), RebindPow2<NewT>()>;
template <typename NewT>
using Repartition =
Simd<NewT, NewN<kPow2, RepartitionLanes<NewT>()>(), kPow2>;
using Half = Simd<T, N, kPow2 - 1>;
using Twice = Simd<T, N, kPow2 + 1>;
};
namespace detail {
template <typename T, size_t N, int kPow2>
constexpr bool IsFull(Simd<T, N, kPow2> ) {
return N == (16 / sizeof(T)) && kPow2 == 0;
}
template <typename T, size_t N, int kPow2>
struct ClampNAndPow2 {
using type = Simd<T, ((N) < (65536) ? (N) : (65536)), ((kPow2) < (3) ? (kPow2) : (3))>;
};
template <typename T, int kPow2>
struct ScalableTagChecker {
using type = typename ClampNAndPow2<T, (16 / sizeof(T)), kPow2>::type;
};
template <typename T, size_t kLimit, int kPow2>
struct CappedTagChecker {
static_assert(kLimit != 0, "Does not make sense to have zero lanes");
static constexpr size_t kLimitPow2 = size_t{1} << hwy::FloorLog2(kLimit);
static constexpr size_t N = ((kLimitPow2) < ((16 / sizeof(T))) ? (kLimitPow2) : ((16 / sizeof(T))));
using type = typename ClampNAndPow2<T, N, kPow2>::type;
};
template <typename T, size_t kNumLanes>
struct FixedTagChecker {
static_assert(kNumLanes != 0, "Does not make sense to have zero lanes");
static_assert(kNumLanes <= (16 / sizeof(T)), "Too many lanes");
using type = Simd<T, kNumLanes, 0>;
};
}
template <typename T, int kPow2 = 0>
using ScalableTag = typename detail::ScalableTagChecker<T, kPow2>::type;
template <typename T, size_t kLimit, int kPow2 = 0>
using CappedTag = typename detail::CappedTagChecker<T, kLimit, kPow2>::type;
template <typename T, size_t kLimit, int kPow2 = 0>
using CappedTagIfFixed = CappedTag<T, kLimit, kPow2>;
template <typename T, size_t kNumLanes>
using FixedTag = typename detail::FixedTagChecker<T, kNumLanes>::type;
template <typename T>
using Full16 = Simd<T, 2 / sizeof(T), 0>;
template <typename T>
using Full32 = Simd<T, 4 / sizeof(T), 0>;
template <typename T>
using Full64 = Simd<T, 8 / sizeof(T), 0>;
template <typename T>
using Full128 = Simd<T, 16 / sizeof(T), 0>;
template <class D>
using TFromD = typename D::T;
template <class D>
inline __attribute__((always_inline)) __attribute__((unused)) constexpr size_t MaxLanes(D) {
return D::kPrivateLanes;
}
template <class D>
inline __attribute__((always_inline)) __attribute__((unused)) constexpr size_t Lanes(D) {
return D::kPrivateLanes;
}
template <class T, class D>
using Rebind = typename D::template Rebind<T>;
template <class D>
using RebindToSigned = Rebind<MakeSigned<TFromD<D>>, D>;
template <class D>
using RebindToUnsigned = Rebind<MakeUnsigned<TFromD<D>>, D>;
template <class D>
using RebindToFloat = Rebind<MakeFloat<TFromD<D>>, D>;
template <class T, class D>
using Repartition = typename D::template Repartition<T>;
template <class D>
using RepartitionToWide = Repartition<MakeWide<TFromD<D>>, D>;
template <class D>
using RepartitionToNarrow = Repartition<MakeNarrow<TFromD<D>>, D>;
template <class D>
using Half = typename D::Half;
template <class D>
using Twice = typename D::Twice;
template <class D>
using BlockDFromD =
Simd<TFromD<D>, ((16 / sizeof(TFromD<D>)) < (D::kPrivateLanes) ? (16 / sizeof(TFromD<D>)) : (D::kPrivateLanes)), 0>;
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC push_options
#pragma GCC target "sse2,ssse3"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSSE3 {
namespace detail {
template <typename T>
struct Raw128 {
using type = __m128i;
};
template <>
struct Raw128<float> {
using type = __m128;
};
template <>
struct Raw128<double> {
using type = __m128d;
};
}
template <typename T, size_t N = 16 / sizeof(T)>
class Vec128 {
using Raw = typename detail::Raw128<T>::type;
public:
using PrivateT = T;
static constexpr size_t kPrivateN = N;
inline __attribute__((always_inline)) Vec128& operator*=(const Vec128 other) {
return *this = (*this * other);
}
inline __attribute__((always_inline)) Vec128& operator/=(const Vec128 other) {
return *this = (*this / other);
}
inline __attribute__((always_inline)) Vec128& operator+=(const Vec128 other) {
return *this = (*this + other);
}
inline __attribute__((always_inline)) Vec128& operator-=(const Vec128 other) {
return *this = (*this - other);
}
inline __attribute__((always_inline)) Vec128& operator&=(const Vec128 other) {
return *this = (*this & other);
}
inline __attribute__((always_inline)) Vec128& operator|=(const Vec128 other) {
return *this = (*this | other);
}
inline __attribute__((always_inline)) Vec128& operator^=(const Vec128 other) {
return *this = (*this ^ other);
}
Raw raw;
};
template <typename T>
using Vec64 = Vec128<T, 8 / sizeof(T)>;
template <typename T>
using Vec32 = Vec128<T, 4 / sizeof(T)>;
template <typename T>
using Vec16 = Vec128<T, 2 / sizeof(T)>;
template <typename T, size_t N = 16 / sizeof(T)>
struct Mask128 {
typename detail::Raw128<T>::type raw;
};
namespace detail {
template <typename T, size_t N>
constexpr uint64_t OnlyActive(uint64_t mask_bits) {
return ((N * sizeof(T)) == 16) ? mask_bits : mask_bits & ((1ull << N) - 1);
}
}
template <class V>
using DFromV = Simd<typename V::PrivateT, V::kPrivateN, 0>;
template <class V>
using TFromV = typename V::PrivateT;
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<!hwy::IsFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<TFromD<D>, D::kPrivateLanes> Zero(D ) {
return Vec128<TFromD<D>, D::kPrivateLanes>{_mm_setzero_si128()};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, D::kPrivateLanes> Zero(D ) {
return Vec128<float, D::kPrivateLanes>{_mm_setzero_ps()};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, D::kPrivateLanes> Zero(D ) {
return Vec128<double, D::kPrivateLanes>{_mm_setzero_pd()};
}
template <class D>
using VFromD = decltype(Zero(D()));
template <class D>
struct Vec2 {
VFromD<D> v0;
VFromD<D> v1;
};
template <class D>
struct Vec3 {
VFromD<D> v0;
VFromD<D> v1;
VFromD<D> v2;
};
template <class D>
struct Vec4 {
VFromD<D> v0;
VFromD<D> v1;
VFromD<D> v2;
VFromD<D> v3;
};
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec2<D> Create2(D , VFromD<D> v0, VFromD<D> v1) {
return Vec2<D>{v0, v1};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec3<D> Create3(D , VFromD<D> v0, VFromD<D> v1, VFromD<D> v2) {
return Vec3<D>{v0, v1, v2};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec4<D> Create4(D , VFromD<D> v0, VFromD<D> v1, VFromD<D> v2,
VFromD<D> v3) {
return Vec4<D>{v0, v1, v2, v3};
}
template <size_t kIndex, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Get2(Vec2<D> tuple) {
static_assert(kIndex < 2, "Tuple index out of bounds");
return kIndex == 0 ? tuple.v0 : tuple.v1;
}
template <size_t kIndex, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Get3(Vec3<D> tuple) {
static_assert(kIndex < 3, "Tuple index out of bounds");
return kIndex == 0 ? tuple.v0 : kIndex == 1 ? tuple.v1 : tuple.v2;
}
template <size_t kIndex, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Get4(Vec4<D> tuple) {
static_assert(kIndex < 4, "Tuple index out of bounds");
return kIndex == 0 ? tuple.v0
: kIndex == 1 ? tuple.v1
: kIndex == 2 ? tuple.v2
: tuple.v3;
}
namespace detail {
inline __attribute__((always_inline)) __m128i BitCastToInteger(__m128i v) { return v; }
inline __attribute__((always_inline)) __m128i BitCastToInteger(__m128 v) { return _mm_castps_si128(v); }
inline __attribute__((always_inline)) __m128i BitCastToInteger(__m128d v) { return _mm_castpd_si128(v); }
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<uint8_t, N * sizeof(T)> BitCastToByte(Vec128<T, N> v) {
return Vec128<uint8_t, N * sizeof(T)>{BitCastToInteger(v.raw)};
}
template <typename T>
struct BitCastFromInteger128 {
inline __attribute__((always_inline)) __m128i operator()(__m128i v) { return v; }
};
template <>
struct BitCastFromInteger128<float> {
inline __attribute__((always_inline)) __m128 operator()(__m128i v) { return _mm_castsi128_ps(v); }
};
template <>
struct BitCastFromInteger128<double> {
inline __attribute__((always_inline)) __m128d operator()(__m128i v) { return _mm_castsi128_pd(v); }
};
template <class D>
inline __attribute__((always_inline)) VFromD<D> BitCastFromByte(D ,
Vec128<uint8_t, D().MaxBytes()> v) {
return VFromD<D>{BitCastFromInteger128<TFromD<D>>()(v.raw)};
}
}
template <class D, typename FromT>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> BitCast(D d,
Vec128<FromT, Repartition<FromT, D>().MaxLanes()> v) {
return detail::BitCastFromByte(d, detail::BitCastToByte(v));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , TFromD<D> t) {
return VFromD<D>{_mm_set1_epi8(static_cast<char>(t))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr,
hwy::EnableIf<!hwy::IsSpecialFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , TFromD<D> t) {
return VFromD<D>{_mm_set1_epi16(static_cast<short>(t))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>() || IsSame<TFromD<D>, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , TFromD<D> t) {
return VFromD<D>{_mm_set1_epi32(static_cast<int>(t))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>() || IsSame<TFromD<D>, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , TFromD<D> t) {
return VFromD<D>{_mm_set1_epi64x(static_cast<long long>(t))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , float t) {
return VFromD<D>{_mm_set1_ps(t)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , double t) {
return VFromD<D>{_mm_set1_pd(t)};
}
template <class D, hwy::EnableIf<hwy::IsSpecialFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D df, TFromD<D> t) {
const RebindToUnsigned<decltype(df)> du;
static_assert(sizeof(TFromD<D>) == 2, "Expecting [b]f16");
uint16_t bits;
CopyBytes<2>(&t, &bits);
return BitCast(df, Set(du, bits));
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wuninitialized"
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<!hwy::IsFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Undefined(D ) {
return VFromD<D>{_mm_undefined_si128()};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Undefined(D ) {
return VFromD<D>{_mm_undefined_ps()};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Undefined(D ) {
return VFromD<D>{_mm_undefined_pd()};
}
#pragma GCC diagnostic pop
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T GetLane(const Vec128<T, N> v) {
return static_cast<T>(_mm_cvtsi128_si32(v.raw) & 0xFF);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T GetLane(const Vec128<T, N> v) {
return static_cast<T>(_mm_cvtsi128_si32(v.raw) & 0xFFFF);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T GetLane(const Vec128<T, N> v) {
return static_cast<T>(_mm_cvtsi128_si32(v.raw));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) float GetLane(const Vec128<float, N> v) {
return _mm_cvtss_f32(v.raw);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T GetLane(const Vec128<T, N> v) {
const DFromV<decltype(v)> d;
alignas(16) T lanes[2];
Store(v, d, lanes);
return lanes[0];
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) double GetLane(const Vec128<double, N> v) {
return _mm_cvtsd_f64(v.raw);
}
template <class D, class FromV, hwy::EnableIf<DFromV<FromV>::kPrivateLanes * sizeof(TFromV<FromV>) <= 16>* = nullptr,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ResizeBitCast(D d, FromV v) {
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, VFromD<decltype(du8)>{detail::BitCastToInteger(v.raw)});
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> And(Vec128<T, N> a, Vec128<T, N> b) {
return Vec128<T, N>{_mm_and_si128(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> And(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_and_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> And(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_and_pd(a.raw, b.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> AndNot(Vec128<T, N> not_mask, Vec128<T, N> mask) {
return Vec128<T, N>{_mm_andnot_si128(not_mask.raw, mask.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> AndNot(const Vec128<float, N> not_mask,
const Vec128<float, N> mask) {
return Vec128<float, N>{_mm_andnot_ps(not_mask.raw, mask.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> AndNot(const Vec128<double, N> not_mask,
const Vec128<double, N> mask) {
return Vec128<double, N>{_mm_andnot_pd(not_mask.raw, mask.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Or(Vec128<T, N> a, Vec128<T, N> b) {
return Vec128<T, N>{_mm_or_si128(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Or(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_or_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Or(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_or_pd(a.raw, b.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Xor(Vec128<T, N> a, Vec128<T, N> b) {
return Vec128<T, N>{_mm_xor_si128(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Xor(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_xor_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Xor(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_xor_pd(a.raw, b.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Not(const Vec128<T, N> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
using VU = VFromD<decltype(du)>;
return Xor(v, BitCast(d, VU{_mm_set1_epi32(-1)}));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Xor3(Vec128<T, N> x1, Vec128<T, N> x2, Vec128<T, N> x3) {
return Xor(x1, Xor(x2, x3));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Or3(Vec128<T, N> o1, Vec128<T, N> o2, Vec128<T, N> o3) {
return Or(o1, Or(o2, o3));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> OrAnd(Vec128<T, N> o, Vec128<T, N> a1, Vec128<T, N> a2) {
return Or(o, And(a1, a2));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfVecThenElse(Vec128<T, N> mask, Vec128<T, N> yes,
Vec128<T, N> no) {
return IfThenElse(MaskFromVec(mask), yes, no);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator&(const Vec128<T, N> a, const Vec128<T, N> b) {
return And(a, b);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator|(const Vec128<T, N> a, const Vec128<T, N> b) {
return Or(a, b);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator^(const Vec128<T, N> a, const Vec128<T, N> b) {
return Xor(a, b);
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Neg(hwy::FloatTag , const Vec128<T, N> v) {
return Xor(v, SignBit(DFromV<decltype(v)>()));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Neg(hwy::NonFloatTag , const Vec128<T, N> v) {
return Zero(DFromV<decltype(v)>()) - v;
}
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Neg(const Vec128<T, N> v) {
return detail::Neg(hwy::IsFloatTag<T>(), v);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Abs(const Vec128<float, N> v) {
const Vec128<int32_t, N> mask{_mm_set1_epi32(0x7FFFFFFF)};
return v & BitCast(DFromV<decltype(v)>(), mask);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Abs(const Vec128<double, N> v) {
const Vec128<int64_t, N> mask{_mm_set1_epi64x(0x7FFFFFFFFFFFFFFFLL)};
return v & BitCast(DFromV<decltype(v)>(), mask);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CopySign(const V magn, const V sign) {
static_assert(IsFloat<TFromV<V>>(), "Only makes sense for floating-point");
const DFromV<decltype(magn)> d;
const auto msb = SignBit(d);
return BitwiseIfThenElse(msb, sign, magn);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CopySignToAbs(const V abs, const V sign) {
const DFromV<decltype(abs)> d;
return OrAnd(abs, SignBit(d), sign);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> MaskFromVec(const Vec128<T, N> v) {
return Mask128<T, N>{v.raw};
}
template <class D>
using MFromD = decltype(MaskFromVec(VFromD<D>()));
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> VecFromMask(const Mask128<T, N> v) {
return Vec128<T, N>{v.raw};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> VecFromMask(D , MFromD<D> v) {
return VFromD<D>{v.raw};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfThenElse(Mask128<T, N> mask, Vec128<T, N> yes,
Vec128<T, N> no) {
const auto vmask = VecFromMask(DFromV<decltype(no)>(), mask);
return Or(And(vmask, yes), AndNot(vmask, no));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfThenElseZero(Mask128<T, N> mask, Vec128<T, N> yes) {
return yes & VecFromMask(DFromV<decltype(yes)>(), mask);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfThenZeroElse(Mask128<T, N> mask, Vec128<T, N> no) {
return AndNot(VecFromMask(DFromV<decltype(no)>(), mask), no);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> Not(const Mask128<T, N> m) {
const Simd<T, N, 0> d;
return MaskFromVec(Not(VecFromMask(d, m)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> And(const Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(And(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> AndNot(const Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(AndNot(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> Or(const Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(Or(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> Xor(const Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(Xor(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> ExclusiveNeither(const Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(AndNot(VecFromMask(d, a), Not(VecFromMask(d, b))));
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> ShiftLeft(const Vec128<uint16_t, N> v) {
return Vec128<uint16_t, N>{_mm_slli_epi16(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> ShiftLeft(const Vec128<uint32_t, N> v) {
return Vec128<uint32_t, N>{_mm_slli_epi32(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> ShiftLeft(const Vec128<uint64_t, N> v) {
return Vec128<uint64_t, N>{_mm_slli_epi64(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> ShiftLeft(const Vec128<int16_t, N> v) {
return Vec128<int16_t, N>{_mm_slli_epi16(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> ShiftLeft(const Vec128<int32_t, N> v) {
return Vec128<int32_t, N>{_mm_slli_epi32(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> ShiftLeft(const Vec128<int64_t, N> v) {
return Vec128<int64_t, N>{_mm_slli_epi64(v.raw, kBits)};
}
template <int kBits, typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftLeft(const Vec128<T, N> v) {
const DFromV<decltype(v)> d8;
const Vec128<T, N> shifted{ShiftLeft<kBits>(Vec128<MakeWide<T>>{v.raw}).raw};
return kBits == 1
? (v + v)
: (shifted & Set(d8, static_cast<T>((0xFF << kBits) & 0xFF)));
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> ShiftRight(const Vec128<uint16_t, N> v) {
return Vec128<uint16_t, N>{_mm_srli_epi16(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> ShiftRight(const Vec128<uint32_t, N> v) {
return Vec128<uint32_t, N>{_mm_srli_epi32(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> ShiftRight(const Vec128<uint64_t, N> v) {
return Vec128<uint64_t, N>{_mm_srli_epi64(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> ShiftRight(const Vec128<uint8_t, N> v) {
const DFromV<decltype(v)> d8;
const Vec128<uint8_t, N> shifted{
ShiftRight<kBits>(Vec128<uint16_t>{v.raw}).raw};
return shifted & Set(d8, 0xFF >> kBits);
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> ShiftRight(const Vec128<int16_t, N> v) {
return Vec128<int16_t, N>{_mm_srai_epi16(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> ShiftRight(const Vec128<int32_t, N> v) {
return Vec128<int32_t, N>{_mm_srai_epi32(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> ShiftRight(const Vec128<int8_t, N> v) {
const DFromV<decltype(v)> di;
const RebindToUnsigned<decltype(di)> du;
const auto shifted = BitCast(di, ShiftRight<kBits>(BitCast(du, v)));
const auto shifted_sign = BitCast(di, Set(du, 0x80 >> kBits));
return (shifted ^ shifted_sign) - shifted_sign;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Load(D , const T* __restrict__ aligned) {
return Vec128<T>{_mm_load_si128(reinterpret_cast<const __m128i*>(aligned))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> Load(D , const float* __restrict__ aligned) {
return Vec128<float>{_mm_load_ps(aligned)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> Load(D , const double* __restrict__ aligned) {
return Vec128<double>{_mm_load_pd(aligned)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> LoadU(D , const T* __restrict__ p) {
return Vec128<T>{_mm_loadu_si128(reinterpret_cast<const __m128i*>(p))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> LoadU(D , const float* __restrict__ p) {
return Vec128<float>{_mm_loadu_ps(p)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> LoadU(D , const double* __restrict__ p) {
return Vec128<double>{_mm_loadu_pd(p)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> Load(D , const T* __restrict__ p) {
return Vec64<T>{_mm_loadl_epi64(reinterpret_cast<const __m128i*>(p))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<float> Load(D , const float* __restrict__ p) {
const __m128 hi = _mm_setzero_ps();
return Vec64<float>{_mm_loadl_pi(hi, reinterpret_cast<const __m64*>(p))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<double> Load(D , const double* __restrict__ p) {
return Vec64<double>{_mm_load_sd(p)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<float> Load(D , const float* __restrict__ p) {
return Vec32<float>{_mm_load_ss(p)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Load(D d, const T* __restrict__ p) {
detail::MaybeUnpoison(p, Lanes(d));
int32_t bits = 0;
CopyBytes<d.MaxBytes()>(p, &bits);
return VFromD<D>{_mm_cvtsi32_si128(bits)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> LoadU(D d, const T* __restrict__ p) {
return Load(d, p);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> LoadDup128(D d, const T* __restrict__ p) {
return LoadU(d, p);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec128<T> v, D , T* __restrict__ aligned) {
_mm_store_si128(reinterpret_cast<__m128i*>(aligned), v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec128<float> v, D , float* __restrict__ aligned) {
_mm_store_ps(aligned, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec128<double> v, D ,
double* __restrict__ aligned) {
_mm_store_pd(aligned, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreU(Vec128<T> v, D , T* __restrict__ p) {
_mm_storeu_si128(reinterpret_cast<__m128i*>(p), v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreU(Vec128<float> v, D , float* __restrict__ p) {
_mm_storeu_ps(p, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreU(Vec128<double> v, D , double* __restrict__ p) {
_mm_storeu_pd(p, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec64<T> v, D , T* __restrict__ p) {
_mm_storel_epi64(reinterpret_cast<__m128i*>(p), v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec64<float> v, D , float* __restrict__ p) {
_mm_storel_pi(reinterpret_cast<__m64*>(p), v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec64<double> v, D , double* __restrict__ p) {
_mm_storel_pd(p, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(VFromD<D> v, D d, T* __restrict__ p) {
CopyBytes<d.MaxBytes()>(&v, p);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec32<float> v, D , float* __restrict__ p) {
_mm_store_ss(p, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreU(VFromD<D> v, D d, T* __restrict__ p) {
Store(v, d, p);
}
template <typename T, size_t N, typename TI, size_t NI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<TI, NI> TableLookupBytes(const Vec128<T, N> bytes,
const Vec128<TI, NI> from) {
return Vec128<TI, NI>{_mm_shuffle_epi8(bytes.raw, from.raw)};
}
template <class V, class VI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VI TableLookupBytesOr0(const V bytes, const VI from) {
return TableLookupBytes(bytes, from);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Shuffle2301(const Vec128<T, N> v) {
static_assert(sizeof(T) == 4, "Only for 32-bit lanes");
static_assert(N == 2 || N == 4, "Does not make sense for N=1");
return Vec128<T, N>{_mm_shuffle_epi32(v.raw, 0xB1)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Shuffle2301(const Vec128<float, N> v) {
static_assert(N == 2 || N == 4, "Does not make sense for N=1");
return Vec128<float, N>{_mm_shuffle_ps(v.raw, v.raw, 0xB1)};
}
namespace detail {
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<T> ShuffleTwo2301(const Vec32<T> a, const Vec32<T> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> d2;
const auto ba = Combine(d2, b, a);
alignas(16) const T kShuffle[8] = {1, 0, 7, 6};
return Vec32<T>{TableLookupBytes(ba, Load(d2, kShuffle)).raw};
}
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ShuffleTwo2301(const Vec64<T> a, const Vec64<T> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> d2;
const auto ba = Combine(d2, b, a);
alignas(16) const T kShuffle[8] = {0x0302, 0x0100, 0x0f0e, 0x0d0c};
return Vec64<T>{TableLookupBytes(ba, Load(d2, kShuffle)).raw};
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ShuffleTwo2301(const Vec128<T> a, const Vec128<T> b) {
const DFromV<decltype(a)> d;
const RebindToFloat<decltype(d)> df;
constexpr int m =
(((
2
) << 6) | ((
3
) << 4) | ((
0
) << 2) | (
1
))
;
return BitCast(d, Vec128<float>{_mm_shuffle_ps(BitCast(df, a).raw,
BitCast(df, b).raw, m)});
}
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<T> ShuffleTwo1230(const Vec32<T> a, const Vec32<T> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> d2;
const auto ba = Combine(d2, b, a);
alignas(16) const T kShuffle[8] = {0, 3, 6, 5};
return Vec32<T>{TableLookupBytes(ba, Load(d2, kShuffle)).raw};
}
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ShuffleTwo1230(const Vec64<T> a, const Vec64<T> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> d2;
const auto ba = Combine(d2, b, a);
alignas(16) const T kShuffle[8] = {0x0100, 0x0706, 0x0d0c, 0x0b0a};
return Vec64<T>{TableLookupBytes(ba, Load(d2, kShuffle)).raw};
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ShuffleTwo1230(const Vec128<T> a, const Vec128<T> b) {
const DFromV<decltype(a)> d;
const RebindToFloat<decltype(d)> df;
constexpr int m =
(((
1
) << 6) | ((
2
) << 4) | ((
3
) << 2) | (
0
))
;
return BitCast(d, Vec128<float>{_mm_shuffle_ps(BitCast(df, a).raw,
BitCast(df, b).raw, m)});
}
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<T> ShuffleTwo3012(const Vec32<T> a, const Vec32<T> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> d2;
const auto ba = Combine(d2, b, a);
alignas(16) const T kShuffle[8] = {2, 1, 4, 7};
return Vec32<T>{TableLookupBytes(ba, Load(d2, kShuffle)).raw};
}
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ShuffleTwo3012(const Vec64<T> a, const Vec64<T> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> d2;
const auto ba = Combine(d2, b, a);
alignas(16) const T kShuffle[8] = {0x0504, 0x0302, 0x0908, 0x0f0e};
return Vec64<T>{TableLookupBytes(ba, Load(d2, kShuffle)).raw};
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ShuffleTwo3012(const Vec128<T> a, const Vec128<T> b) {
const DFromV<decltype(a)> d;
const RebindToFloat<decltype(d)> df;
constexpr int m =
(((
3
) << 6) | ((
0
) << 4) | ((
1
) << 2) | (
2
))
;
return BitCast(d, Vec128<float>{_mm_shuffle_ps(BitCast(df, a).raw,
BitCast(df, b).raw, m)});
}
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> Shuffle1032(const Vec128<uint32_t> v) {
return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x4E)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> Shuffle1032(const Vec128<int32_t> v) {
return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x4E)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> Shuffle1032(const Vec128<float> v) {
return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x4E)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t> Shuffle01(const Vec128<uint64_t> v) {
return Vec128<uint64_t>{_mm_shuffle_epi32(v.raw, 0x4E)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t> Shuffle01(const Vec128<int64_t> v) {
return Vec128<int64_t>{_mm_shuffle_epi32(v.raw, 0x4E)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> Shuffle01(const Vec128<double> v) {
return Vec128<double>{_mm_shuffle_pd(v.raw, v.raw, 1)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> Shuffle0321(const Vec128<uint32_t> v) {
return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x39)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> Shuffle0321(const Vec128<int32_t> v) {
return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x39)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> Shuffle0321(const Vec128<float> v) {
return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x39)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> Shuffle2103(const Vec128<uint32_t> v) {
return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x93)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> Shuffle2103(const Vec128<int32_t> v) {
return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x93)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> Shuffle2103(const Vec128<float> v) {
return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x93)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> Shuffle0123(const Vec128<uint32_t> v) {
return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x1B)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> Shuffle0123(const Vec128<int32_t> v) {
return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x1B)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> Shuffle0123(const Vec128<float> v) {
return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x1B)};
}
template <class DTo, typename TFrom, size_t NFrom>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<DTo> RebindMask(DTo dto, Mask128<TFrom, NFrom> m) {
static_assert(sizeof(TFrom) == sizeof(TFromD<DTo>), "Must have same size");
const Simd<TFrom, NFrom, 0> d;
return MaskFromVec(BitCast(dto, VecFromMask(d, m)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> TestBit(Vec128<T, N> v, Vec128<T, N> bit) {
static_assert(!hwy::IsFloat<T>(), "Only integer vectors supported");
return (v & bit) == bit;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint8_t, N> operator==(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Mask128<uint8_t, N>{_mm_cmpeq_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint16_t, N> operator==(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Mask128<uint16_t, N>{_mm_cmpeq_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint32_t, N> operator==(const Vec128<uint32_t, N> a,
const Vec128<uint32_t, N> b) {
return Mask128<uint32_t, N>{_mm_cmpeq_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint64_t, N> operator==(const Vec128<uint64_t, N> a,
const Vec128<uint64_t, N> b) {
const DFromV<decltype(a)> d64;
const RepartitionToNarrow<decltype(d64)> d32;
const auto cmp32 = VecFromMask(d32, Eq(BitCast(d32, a), BitCast(d32, b)));
const auto cmp64 = cmp32 & Shuffle2301(cmp32);
return MaskFromVec(BitCast(d64, cmp64));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int8_t, N> operator==(const Vec128<int8_t, N> a,
const Vec128<int8_t, N> b) {
return Mask128<int8_t, N>{_mm_cmpeq_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int16_t, N> operator==(Vec128<int16_t, N> a,
Vec128<int16_t, N> b) {
return Mask128<int16_t, N>{_mm_cmpeq_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int32_t, N> operator==(const Vec128<int32_t, N> a,
const Vec128<int32_t, N> b) {
return Mask128<int32_t, N>{_mm_cmpeq_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int64_t, N> operator==(const Vec128<int64_t, N> a,
const Vec128<int64_t, N> b) {
const DFromV<decltype(a)> d;
RebindToUnsigned<decltype(d)> du;
return RebindMask(d, BitCast(du, a) == BitCast(du, b));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<float, N> operator==(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Mask128<float, N>{_mm_cmpeq_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<double, N> operator==(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Mask128<double, N>{_mm_cmpeq_pd(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint8_t, N> operator!=(Vec128<uint8_t, N> a,
Vec128<uint8_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint16_t, N> operator!=(Vec128<uint16_t, N> a,
Vec128<uint16_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint32_t, N> operator!=(Vec128<uint32_t, N> a,
Vec128<uint32_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint64_t, N> operator!=(Vec128<uint64_t, N> a,
Vec128<uint64_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int8_t, N> operator!=(Vec128<int8_t, N> a,
Vec128<int8_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int16_t, N> operator!=(Vec128<int16_t, N> a,
Vec128<int16_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int32_t, N> operator!=(Vec128<int32_t, N> a,
Vec128<int32_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int64_t, N> operator!=(Vec128<int64_t, N> a,
Vec128<int64_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<float, N> operator!=(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Mask128<float, N>{_mm_cmpneq_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<double, N> operator!=(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Mask128<double, N>{_mm_cmpneq_pd(a.raw, b.raw)};
}
namespace detail {
template <size_t N>
inline __attribute__((always_inline)) Mask128<int8_t, N> Gt(hwy::SignedTag , Vec128<int8_t, N> a,
Vec128<int8_t, N> b) {
return Mask128<int8_t, N>{_mm_cmpgt_epi8(a.raw, b.raw)};
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<int16_t, N> Gt(hwy::SignedTag , Vec128<int16_t, N> a,
Vec128<int16_t, N> b) {
return Mask128<int16_t, N>{_mm_cmpgt_epi16(a.raw, b.raw)};
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<int32_t, N> Gt(hwy::SignedTag , Vec128<int32_t, N> a,
Vec128<int32_t, N> b) {
return Mask128<int32_t, N>{_mm_cmpgt_epi32(a.raw, b.raw)};
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<int64_t, N> Gt(hwy::SignedTag ,
const Vec128<int64_t, N> a,
const Vec128<int64_t, N> b) {
const DFromV<decltype(a)> d;
const RepartitionToNarrow<decltype(d)> d32;
const Vec128<int64_t, N> m_eq32{Eq(BitCast(d32, a), BitCast(d32, b)).raw};
const Vec128<int64_t, N> m_gt32{Gt(BitCast(d32, a), BitCast(d32, b)).raw};
const __m128i upper = OrAnd(m_gt32, m_eq32, Sub(b, a)).raw;
return Mask128<int64_t, N>{_mm_shuffle_epi32(upper,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Mask128<T, N> Gt(hwy::UnsignedTag , Vec128<T, N> a,
Vec128<T, N> b) {
const DFromV<decltype(a)> du;
const RebindToSigned<decltype(du)> di;
const Vec128<T, N> msb = Set(du, (LimitsMax<T>() >> 1) + 1);
const auto sa = BitCast(di, Xor(a, msb));
const auto sb = BitCast(di, Xor(b, msb));
return RebindMask(du, Gt(hwy::SignedTag(), sa, sb));
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<float, N> Gt(hwy::FloatTag , Vec128<float, N> a,
Vec128<float, N> b) {
return Mask128<float, N>{_mm_cmpgt_ps(a.raw, b.raw)};
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<double, N> Gt(hwy::FloatTag , Vec128<double, N> a,
Vec128<double, N> b) {
return Mask128<double, N>{_mm_cmpgt_pd(a.raw, b.raw)};
}
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Mask128<T, N> operator>(Vec128<T, N> a, Vec128<T, N> b) {
return detail::Gt(hwy::TypeTag<T>(), a, b);
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) Mask128<T, N> Ge(hwy::SignedTag tag, Vec128<T, N> a,
Vec128<T, N> b) {
return Not(Gt(tag, b, a));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Mask128<T, N> Ge(hwy::UnsignedTag tag, Vec128<T, N> a,
Vec128<T, N> b) {
return Not(Gt(tag, b, a));
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<float, N> Ge(hwy::FloatTag , Vec128<float, N> a,
Vec128<float, N> b) {
return Mask128<float, N>{_mm_cmpge_ps(a.raw, b.raw)};
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<double, N> Ge(hwy::FloatTag , Vec128<double, N> a,
Vec128<double, N> b) {
return Mask128<double, N>{_mm_cmpge_pd(a.raw, b.raw)};
}
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> operator>=(Vec128<T, N> a, Vec128<T, N> b) {
return detail::Ge(hwy::TypeTag<T>(), a, b);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> operator<(Vec128<T, N> a, Vec128<T, N> b) {
return b > a;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> operator<=(Vec128<T, N> a, Vec128<T, N> b) {
return b >= a;
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{_mm_set_epi8(
static_cast<char>(15), static_cast<char>(14), static_cast<char>(13),
static_cast<char>(12), static_cast<char>(11), static_cast<char>(10),
static_cast<char>(9), static_cast<char>(8), static_cast<char>(7),
static_cast<char>(6), static_cast<char>(5), static_cast<char>(4),
static_cast<char>(3), static_cast<char>(2), static_cast<char>(1),
static_cast<char>(0))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr,
hwy::EnableIf<!hwy::IsSpecialFloat<TFromD<D> >()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{_mm_set_epi16(int16_t{7}, int16_t{6}, int16_t{5}, int16_t{4},
int16_t{3}, int16_t{2}, int16_t{1},
int16_t{0})};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>() || IsSame<TFromD<D>, int32_t>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{
_mm_set_epi32(int32_t{3}, int32_t{2}, int32_t{1}, int32_t{0})};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>() || IsSame<TFromD<D>, int64_t>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{_mm_set_epi64x(int64_t{1}, int64_t{0})};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{_mm_set_ps(3.0f, 2.0f, 1.0f, 0.0f)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{_mm_set_pd(1.0, 0.0)};
}
}
template <class D, typename T2, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Iota(D d, const T2 first) {
const auto result_iota =
detail::Iota0(d) + Set(d, static_cast<TFromD<D>>(first));
return result_iota;
}
template <class D, class M = MFromD<D>, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) M FirstN(D d, size_t num) {
const RebindToSigned<decltype(d)> di;
using TI = TFromD<decltype(di)>;
return RebindMask(d, detail::Iota0(di) < Set(di, static_cast<TI>(num)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> MaskedLoad(MFromD<D> m, D d, const T* __restrict__ p) {
return IfThenElseZero(m, LoadU(d, p));
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> MaskedLoadOr(VFromD<D> v, MFromD<D> m, D d,
const T* __restrict__ p) {
return IfThenElse(m, LoadU(d, p), v);
}
namespace detail {
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void ScalarMaskedStore(VFromD<D> v, MFromD<D> m, D d,
TFromD<D>* __restrict__ p) {
const RebindToSigned<decltype(d)> di;
using TI = TFromD<decltype(di)>;
alignas(16) TI buf[MaxLanes(d)];
alignas(16) TI mask[MaxLanes(d)];
Store(BitCast(di, v), di, buf);
Store(BitCast(di, VecFromMask(d, m)), di, mask);
for (size_t i = 0; i < MaxLanes(d); ++i) {
if (mask[i]) {
CopySameSize(buf + i, p + i);
}
}
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void BlendedStore(VFromD<D> v, MFromD<D> m, D d,
TFromD<D>* __restrict__ p) {
detail::ScalarMaskedStore(v, m, d, p);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> operator+(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_add_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> operator+(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_add_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> operator+(const Vec128<uint32_t, N> a,
const Vec128<uint32_t, N> b) {
return Vec128<uint32_t, N>{_mm_add_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> operator+(const Vec128<uint64_t, N> a,
const Vec128<uint64_t, N> b) {
return Vec128<uint64_t, N>{_mm_add_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> operator+(const Vec128<int8_t, N> a,
const Vec128<int8_t, N> b) {
return Vec128<int8_t, N>{_mm_add_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> operator+(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_add_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> operator+(const Vec128<int32_t, N> a,
const Vec128<int32_t, N> b) {
return Vec128<int32_t, N>{_mm_add_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> operator+(const Vec128<int64_t, N> a,
const Vec128<int64_t, N> b) {
return Vec128<int64_t, N>{_mm_add_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> operator+(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_add_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> operator+(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_add_pd(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> operator-(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_sub_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> operator-(Vec128<uint16_t, N> a,
Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_sub_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> operator-(const Vec128<uint32_t, N> a,
const Vec128<uint32_t, N> b) {
return Vec128<uint32_t, N>{_mm_sub_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> operator-(const Vec128<uint64_t, N> a,
const Vec128<uint64_t, N> b) {
return Vec128<uint64_t, N>{_mm_sub_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> operator-(const Vec128<int8_t, N> a,
const Vec128<int8_t, N> b) {
return Vec128<int8_t, N>{_mm_sub_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> operator-(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_sub_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> operator-(const Vec128<int32_t, N> a,
const Vec128<int32_t, N> b) {
return Vec128<int32_t, N>{_mm_sub_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> operator-(const Vec128<int64_t, N> a,
const Vec128<int64_t, N> b) {
return Vec128<int64_t, N>{_mm_sub_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> operator-(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_sub_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> operator-(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_sub_pd(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N / 8> SumsOf8(const Vec128<uint8_t, N> v) {
return Vec128<uint64_t, N / 8>{_mm_sad_epu8(v.raw, _mm_setzero_si128())};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N / 8> SumsOf8AbsDiff(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint64_t, N / 8>{_mm_sad_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> SaturatedAdd(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_adds_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> SaturatedAdd(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_adds_epu16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> SaturatedAdd(const Vec128<int8_t, N> a,
const Vec128<int8_t, N> b) {
return Vec128<int8_t, N>{_mm_adds_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> SaturatedAdd(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_adds_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> SaturatedSub(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_subs_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> SaturatedSub(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_subs_epu16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> SaturatedSub(const Vec128<int8_t, N> a,
const Vec128<int8_t, N> b) {
return Vec128<int8_t, N>{_mm_subs_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> SaturatedSub(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_subs_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> AverageRound(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_avg_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> AverageRound(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_avg_epu16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> operator*(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_mullo_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> operator*(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_mullo_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> MulHigh(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_mulhi_epu16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> MulHigh(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_mulhi_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, (N + 1) / 2> MulEven(const Vec128<uint32_t, N> a,
const Vec128<uint32_t, N> b) {
return Vec128<uint64_t, (N + 1) / 2>{_mm_mul_epu32(a.raw, b.raw)};
}
template <size_t N, hwy::EnableIf<N * sizeof(int32_t) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, (N + 1) / 2> MulEven(const Vec128<int32_t, N> a,
const Vec128<int32_t, N> b) {
const DFromV<decltype(a)> d;
const RepartitionToWide<decltype(d)> dw;
return Set(dw, static_cast<int64_t>(GetLane(a)) * GetLane(b));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t> MulEven(Vec128<int32_t> a, Vec128<int32_t> b) {
alignas(16) int32_t a_lanes[4];
alignas(16) int32_t b_lanes[4];
const DFromV<decltype(a)> di32;
const RepartitionToWide<decltype(di32)> di64;
Store(a, di32, a_lanes);
Store(b, di32, b_lanes);
alignas(16) int64_t mul[2];
mul[0] = static_cast<int64_t>(a_lanes[0]) * b_lanes[0];
mul[1] = static_cast<int64_t>(a_lanes[2]) * b_lanes[2];
return Load(di64, mul);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> operator*(const Vec128<uint32_t, N> a,
const Vec128<uint32_t, N> b) {
const __m128i a_x3x1 = _mm_shuffle_epi32(a.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
);
const auto mullo_x2x0 = MulEven(a, b);
const __m128i b_x3x1 = _mm_shuffle_epi32(b.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
);
const auto mullo_x3x1 =
MulEven(Vec128<uint32_t, N>{a_x3x1}, Vec128<uint32_t, N>{b_x3x1});
const __m128i mul_20 =
_mm_shuffle_epi32(mullo_x2x0.raw,
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
);
const __m128i mul_31 =
_mm_shuffle_epi32(mullo_x3x1.raw,
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
);
return Vec128<uint32_t, N>{_mm_unpacklo_epi32(mul_20, mul_31)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> operator*(const Vec128<int32_t, N> a,
const Vec128<int32_t, N> b) {
const DFromV<decltype(a)> d;
const RebindToUnsigned<decltype(d)> du;
return BitCast(d, BitCast(du, a) * BitCast(du, b));
}
template <int kBits, typename T, size_t N,
hwy::EnableIf<((size_t{1} << sizeof(T)) & ((1 << 1) | (1 << 2))) != 0>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> RotateRight(const Vec128<T, N> v) {
constexpr size_t kSizeInBits = sizeof(T) * 8;
static_assert(0 <= kBits && kBits < kSizeInBits, "Invalid shift count");
if (kBits == 0) return v;
return Or(ShiftRight<kBits>(v),
ShiftLeft<((kSizeInBits - 1) < (kSizeInBits - kBits) ? (kSizeInBits - 1) : (kSizeInBits - kBits))>(v));
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> RotateRight(const Vec128<uint32_t, N> v) {
static_assert(0 <= kBits && kBits < 32, "Invalid shift count");
if (kBits == 0) return v;
return Or(ShiftRight<kBits>(v), ShiftLeft<((31) < (32 - kBits) ? (31) : (32 - kBits))>(v));
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> RotateRight(const Vec128<uint64_t, N> v) {
static_assert(0 <= kBits && kBits < 64, "Invalid shift count");
if (kBits == 0) return v;
return Or(ShiftRight<kBits>(v), ShiftLeft<((63) < (64 - kBits) ? (63) : (64 - kBits))>(v));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> BroadcastSignBit(const Vec128<int8_t, N> v) {
const DFromV<decltype(v)> d;
return VecFromMask(v < Zero(d));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> BroadcastSignBit(const Vec128<int16_t, N> v) {
return ShiftRight<15>(v);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> BroadcastSignBit(const Vec128<int32_t, N> v) {
return ShiftRight<31>(v);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> BroadcastSignBit(const Vec128<int64_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToNarrow<decltype(d)> d32;
const auto sign = ShiftRight<31>(BitCast(d32, v));
return Vec128<int64_t, N>{
_mm_shuffle_epi32(sign.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> Abs(const Vec128<int8_t, N> v) {
return Vec128<int8_t, N>{_mm_abs_epi8(v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> Abs(const Vec128<int16_t, N> v) {
return Vec128<int16_t, N>{_mm_abs_epi16(v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> Abs(const Vec128<int32_t, N> v) {
return Vec128<int32_t, N>{_mm_abs_epi32(v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> Abs(const Vec128<int64_t, N> v) {
const auto zero = Zero(DFromV<decltype(v)>());
return IfThenElse(MaskFromVec(BroadcastSignBit(v)), zero - v, v);
}
using Shift64Count = int;
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> ShiftRight(const Vec128<int64_t, N> v) {
const DFromV<decltype(v)> di;
const RebindToUnsigned<decltype(di)> du;
const auto right = BitCast(di, ShiftRight<kBits>(BitCast(du, v)));
const auto sign = ShiftLeft<64 - kBits>(BroadcastSignBit(v));
return right | sign;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ZeroIfNegative(Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only works for float");
const DFromV<decltype(v)> d;
const RebindToSigned<decltype(d)> di;
const auto mask = MaskFromVec(BitCast(d, BroadcastSignBit(BitCast(di, v))));
return IfThenElse(mask, Zero(d), v);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> IfNegativeThenElse(const Vec128<int8_t, N> v,
const Vec128<int8_t, N> yes,
const Vec128<int8_t, N> no) {
return IfThenElse(MaskFromVec(v), yes, no);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfNegativeThenElse(Vec128<T, N> v, Vec128<T, N> yes,
Vec128<T, N> no) {
static_assert(IsSigned<T>(), "Only works for signed/float");
const DFromV<decltype(v)> d;
const RebindToSigned<decltype(d)> di;
v = BitCast(d, BroadcastSignBit(BitCast(di, v)));
return IfThenElse(MaskFromVec(v), yes, no);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) != (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfNegativeThenElse(Vec128<T, N> v, Vec128<T, N> yes,
Vec128<T, N> no) {
static_assert(IsSigned<T>(), "Only works for signed/float");
const DFromV<decltype(v)> d;
const RebindToFloat<decltype(d)> df;
return BitCast(d, IfThenElse(MaskFromVec(BitCast(df, v)), BitCast(df, yes),
BitCast(df, no)));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> ShiftLeftSame(const Vec128<uint16_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint16_t, N>{_mm_slli_epi16(v.raw, bits)};
}
return Vec128<uint16_t, N>{_mm_sll_epi16(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> ShiftLeftSame(const Vec128<uint32_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint32_t, N>{_mm_slli_epi32(v.raw, bits)};
}
return Vec128<uint32_t, N>{_mm_sll_epi32(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> ShiftLeftSame(const Vec128<uint64_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint64_t, N>{_mm_slli_epi64(v.raw, bits)};
}
return Vec128<uint64_t, N>{_mm_sll_epi64(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> ShiftLeftSame(const Vec128<int16_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<int16_t, N>{_mm_slli_epi16(v.raw, bits)};
}
return Vec128<int16_t, N>{_mm_sll_epi16(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> ShiftLeftSame(const Vec128<int32_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<int32_t, N>{_mm_slli_epi32(v.raw, bits)};
}
return Vec128<int32_t, N>{_mm_sll_epi32(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> ShiftLeftSame(const Vec128<int64_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<int64_t, N>{_mm_slli_epi64(v.raw, bits)};
}
return Vec128<int64_t, N>{_mm_sll_epi64(v.raw, _mm_cvtsi32_si128(bits))};
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftLeftSame(const Vec128<T, N> v, const int bits) {
const DFromV<decltype(v)> d8;
const Vec128<T, N> shifted{
ShiftLeftSame(Vec128<MakeWide<T>>{v.raw}, bits).raw};
return shifted & Set(d8, static_cast<T>((0xFF << bits) & 0xFF));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> ShiftRightSame(const Vec128<uint16_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint16_t, N>{_mm_srli_epi16(v.raw, bits)};
}
return Vec128<uint16_t, N>{_mm_srl_epi16(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> ShiftRightSame(const Vec128<uint32_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint32_t, N>{_mm_srli_epi32(v.raw, bits)};
}
return Vec128<uint32_t, N>{_mm_srl_epi32(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> ShiftRightSame(const Vec128<uint64_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint64_t, N>{_mm_srli_epi64(v.raw, bits)};
}
return Vec128<uint64_t, N>{_mm_srl_epi64(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> ShiftRightSame(Vec128<uint8_t, N> v,
const int bits) {
const DFromV<decltype(v)> d8;
const Vec128<uint8_t, N> shifted{
ShiftRightSame(Vec128<uint16_t>{v.raw}, bits).raw};
return shifted & Set(d8, static_cast<uint8_t>(0xFF >> bits));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> ShiftRightSame(const Vec128<int16_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<int16_t, N>{_mm_srai_epi16(v.raw, bits)};
}
return Vec128<int16_t, N>{_mm_sra_epi16(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> ShiftRightSame(const Vec128<int32_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<int32_t, N>{_mm_srai_epi32(v.raw, bits)};
}
return Vec128<int32_t, N>{_mm_sra_epi32(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> ShiftRightSame(const Vec128<int64_t, N> v,
const int bits) {
const DFromV<decltype(v)> di;
const RebindToUnsigned<decltype(di)> du;
const auto right = BitCast(di, ShiftRightSame(BitCast(du, v), bits));
const auto sign = ShiftLeftSame(BroadcastSignBit(v), 64 - bits);
return right | sign;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> ShiftRightSame(Vec128<int8_t, N> v, const int bits) {
const DFromV<decltype(v)> di;
const RebindToUnsigned<decltype(di)> du;
const auto shifted = BitCast(di, ShiftRightSame(BitCast(du, v), bits));
const auto shifted_sign =
BitCast(di, Set(du, static_cast<uint8_t>(0x80 >> bits)));
return (shifted ^ shifted_sign) - shifted_sign;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> operator*(Vec128<float, N> a, Vec128<float, N> b) {
return Vec128<float, N>{_mm_mul_ps(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, 1> operator*(const Vec128<float, 1> a,
const Vec128<float, 1> b) {
return Vec128<float, 1>{_mm_mul_ss(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> operator*(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_mul_pd(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<double> operator*(const Vec64<double> a, const Vec64<double> b) {
return Vec64<double>{_mm_mul_sd(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> operator/(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_div_ps(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, 1> operator/(const Vec128<float, 1> a,
const Vec128<float, 1> b) {
return Vec128<float, 1>{_mm_div_ss(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> operator/(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_div_pd(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<double> operator/(const Vec64<double> a, const Vec64<double> b) {
return Vec64<double>{_mm_div_sd(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> ApproximateReciprocal(const Vec128<float, N> v) {
return Vec128<float, N>{_mm_rcp_ps(v.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, 1> ApproximateReciprocal(const Vec128<float, 1> v) {
return Vec128<float, 1>{_mm_rcp_ss(v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> AbsDiff(Vec128<float, N> a, Vec128<float, N> b) {
return Abs(a - b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> MulAdd(Vec128<float, N> mul, Vec128<float, N> x,
Vec128<float, N> add) {
return mul * x + add;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> MulAdd(Vec128<double, N> mul, Vec128<double, N> x,
Vec128<double, N> add) {
return mul * x + add;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> NegMulAdd(Vec128<float, N> mul, Vec128<float, N> x,
Vec128<float, N> add) {
return add - mul * x;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> NegMulAdd(Vec128<double, N> mul, Vec128<double, N> x,
Vec128<double, N> add) {
return add - mul * x;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> MulSub(Vec128<float, N> mul, Vec128<float, N> x,
Vec128<float, N> sub) {
return mul * x - sub;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> MulSub(Vec128<double, N> mul, Vec128<double, N> x,
Vec128<double, N> sub) {
return mul * x - sub;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> NegMulSub(Vec128<float, N> mul, Vec128<float, N> x,
Vec128<float, N> sub) {
return Neg(mul) * x - sub;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> NegMulSub(Vec128<double, N> mul, Vec128<double, N> x,
Vec128<double, N> sub) {
return Neg(mul) * x - sub;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Sqrt(Vec128<float, N> v) {
return Vec128<float, N>{_mm_sqrt_ps(v.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, 1> Sqrt(Vec128<float, 1> v) {
return Vec128<float, 1>{_mm_sqrt_ss(v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Sqrt(Vec128<double, N> v) {
return Vec128<double, N>{_mm_sqrt_pd(v.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<double> Sqrt(Vec64<double> v) {
return Vec64<double>{_mm_sqrt_sd(_mm_setzero_pd(), v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> ApproximateReciprocalSqrt(Vec128<float, N> v) {
return Vec128<float, N>{_mm_rsqrt_ps(v.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, 1> ApproximateReciprocalSqrt(Vec128<float, 1> v) {
return Vec128<float, 1>{_mm_rsqrt_ss(v.raw)};
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) __attribute__((unused)) Vec128<T, N> MinU(const Vec128<T, N> a,
const Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const RebindToUnsigned<decltype(d)> du;
const RebindToSigned<decltype(d)> di;
const auto msb = Set(du, static_cast<T>(T(1) << (sizeof(T) * 8 - 1)));
const auto gt = RebindMask(du, BitCast(di, a ^ msb) > BitCast(di, b ^ msb));
return IfThenElse(gt, b, a);
}
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> Min(Vec128<uint8_t, N> a, Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_min_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> Min(Vec128<uint16_t, N> a, Vec128<uint16_t, N> b) {
return detail::MinU(a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> Min(Vec128<uint32_t, N> a, Vec128<uint32_t, N> b) {
return detail::MinU(a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> Min(Vec128<uint64_t, N> a, Vec128<uint64_t, N> b) {
return detail::MinU(a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> Min(Vec128<int8_t, N> a, Vec128<int8_t, N> b) {
return IfThenElse(a < b, a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> Min(Vec128<int16_t, N> a, Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_min_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> Min(Vec128<int32_t, N> a, Vec128<int32_t, N> b) {
return IfThenElse(a < b, a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> Min(Vec128<int64_t, N> a, Vec128<int64_t, N> b) {
return IfThenElse(a < b, a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Min(Vec128<float, N> a, Vec128<float, N> b) {
return Vec128<float, N>{_mm_min_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Min(Vec128<double, N> a, Vec128<double, N> b) {
return Vec128<double, N>{_mm_min_pd(a.raw, b.raw)};
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) __attribute__((unused)) Vec128<T, N> MaxU(const Vec128<T, N> a,
const Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const RebindToUnsigned<decltype(d)> du;
const RebindToSigned<decltype(d)> di;
const auto msb = Set(du, static_cast<T>(T(1) << (sizeof(T) * 8 - 1)));
const auto gt = RebindMask(du, BitCast(di, a ^ msb) > BitCast(di, b ^ msb));
return IfThenElse(gt, a, b);
}
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> Max(Vec128<uint8_t, N> a, Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_max_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> Max(Vec128<uint16_t, N> a, Vec128<uint16_t, N> b) {
return detail::MaxU(a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> Max(Vec128<uint32_t, N> a, Vec128<uint32_t, N> b) {
return detail::MaxU(a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> Max(Vec128<uint64_t, N> a, Vec128<uint64_t, N> b) {
return detail::MaxU(a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> Max(Vec128<int8_t, N> a, Vec128<int8_t, N> b) {
return IfThenElse(a < b, b, a);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> Max(Vec128<int16_t, N> a, Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_max_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> Max(Vec128<int32_t, N> a, Vec128<int32_t, N> b) {
return IfThenElse(a < b, b, a);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> Max(Vec128<int64_t, N> a, Vec128<int64_t, N> b) {
return IfThenElse(a < b, b, a);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Max(Vec128<float, N> a, Vec128<float, N> b) {
return Vec128<float, N>{_mm_max_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Max(Vec128<double, N> a, Vec128<double, N> b) {
return Vec128<double, N>{_mm_max_pd(a.raw, b.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<!hwy::IsFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Stream(VFromD<D> v, D , TFromD<D>* __restrict__ aligned) {
_mm_stream_si128(reinterpret_cast<__m128i*>(aligned), v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Stream(VFromD<D> v, D , float* __restrict__ aligned) {
_mm_stream_ps(aligned, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Stream(VFromD<D> v, D , double* __restrict__ aligned) {
_mm_stream_pd(aligned, v.raw);
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-conversion"
using GatherIndex64 = long long int;
static_assert(sizeof(GatherIndex64) == 8, "Must be 64-bit type");
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>, class VI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void ScatterOffset(VFromD<D> v, D d, T* __restrict__ base, VI offset) {
using TI = TFromV<VI>;
static_assert(sizeof(T) == sizeof(TI), "Index/lane size must match");
alignas(16) T lanes[MaxLanes(d)];
Store(v, d, lanes);
alignas(16) TI offset_lanes[MaxLanes(d)];
Store(offset, Rebind<TI, decltype(d)>(), offset_lanes);
uint8_t* base_bytes = reinterpret_cast<uint8_t*>(base);
for (size_t i = 0; i < MaxLanes(d); ++i) {
CopyBytes<sizeof(T)>(&lanes[i], base_bytes + offset_lanes[i]);
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>, class VI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void ScatterIndex(VFromD<D> v, D d, T* __restrict__ base, VI index) {
using TI = TFromV<VI>;
static_assert(sizeof(T) == sizeof(TI), "Index/lane size must match");
alignas(16) T lanes[MaxLanes(d)];
Store(v, d, lanes);
alignas(16) TI index_lanes[MaxLanes(d)];
Store(index, Rebind<TI, decltype(d)>(), index_lanes);
for (size_t i = 0; i < MaxLanes(d); ++i) {
base[index_lanes[i]] = lanes[i];
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>, class VI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> GatherOffset(D d, const T* __restrict__ base, VI offset) {
using TI = TFromV<VI>;
static_assert(sizeof(T) == sizeof(TI), "Index/lane size must match");
alignas(16) TI offset_lanes[MaxLanes(d)];
Store(offset, Rebind<TI, decltype(d)>(), offset_lanes);
alignas(16) T lanes[MaxLanes(d)];
const uint8_t* base_bytes = reinterpret_cast<const uint8_t*>(base);
for (size_t i = 0; i < MaxLanes(d); ++i) {
CopyBytes<sizeof(T)>(base_bytes + offset_lanes[i], &lanes[i]);
}
return Load(d, lanes);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>, class VI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> GatherIndex(D d, const T* __restrict__ base, VI index) {
using TI = TFromV<VI>;
static_assert(sizeof(T) == sizeof(TI), "Index/lane size must match");
alignas(16) TI index_lanes[MaxLanes(d)];
Store(index, Rebind<TI, decltype(d)>(), index_lanes);
alignas(16) T lanes[MaxLanes(d)];
for (size_t i = 0; i < MaxLanes(d); ++i) {
lanes[i] = base[index_lanes[i]];
}
return Load(d, lanes);
}
#pragma GCC diagnostic pop
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> LowerHalf(D , VFromD<Twice<D>> v) {
return VFromD<D>{v.raw};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N / 2> LowerHalf(Vec128<T, N> v) {
return Vec128<T, N / 2>{v.raw};
}
template <int kBytes, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ShiftLeftBytes(D d, VFromD<D> v) {
static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes");
const RebindToUnsigned<decltype(d)> du;
return BitCast(
d, VFromD<decltype(du)>{_mm_slli_si128(BitCast(du, v).raw, kBytes)});
}
template <int kBytes, typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftLeftBytes(const Vec128<T, N> v) {
return ShiftLeftBytes<kBytes>(DFromV<decltype(v)>(), v);
}
template <int kLanes, class D, typename T = TFromD<D>,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ShiftLeftLanes(D d, const VFromD<D> v) {
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, ShiftLeftBytes<kLanes * sizeof(T)>(BitCast(d8, v)));
}
template <int kLanes, typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftLeftLanes(const Vec128<T, N> v) {
return ShiftLeftLanes<kLanes>(DFromV<decltype(v)>(), v);
}
template <int kBytes, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ShiftRightBytes(D d, VFromD<D> v) {
static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes");
const RebindToUnsigned<decltype(d)> du;
if (d.MaxBytes() != 16) {
const Full128<TFromD<D>> dfull;
const VFromD<decltype(dfull)> vfull{v.raw};
v = VFromD<D>{IfThenElseZero(FirstN(dfull, MaxLanes(d)), vfull).raw};
}
return BitCast(
d, VFromD<decltype(du)>{_mm_srli_si128(BitCast(du, v).raw, kBytes)});
}
template <int kLanes, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ShiftRightLanes(D d, const VFromD<D> v) {
const Repartition<uint8_t, decltype(d)> d8;
constexpr size_t kBytes = kLanes * sizeof(TFromD<D>);
return BitCast(d, ShiftRightBytes<kBytes>(d8, BitCast(d8, v)));
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> UpperHalf(D , Vec128<T> v) {
return Vec64<T>{_mm_unpackhi_epi64(v.raw, v.raw)};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<float> UpperHalf(D , Vec128<float> v) {
return Vec64<float>{_mm_movehl_ps(v.raw, v.raw)};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<double> UpperHalf(D , Vec128<double> v) {
return Vec64<double>{_mm_unpackhi_pd(v.raw, v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> UpperHalf(D d, VFromD<Twice<D>> v) {
return LowerHalf(d, ShiftRightBytes<d.MaxBytes()>(Twice<D>(), v));
}
namespace detail {
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
inline __attribute__((always_inline)) T ExtractLane(const Vec128<T, N> v) {
static_assert(kLane < N, "Lane index out of bounds");
const int pair = _mm_extract_epi16(v.raw, kLane / 2);
constexpr int kShift = kLane & 1 ? 8 : 0;
return static_cast<T>((pair >> kShift) & 0xFF);
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
inline __attribute__((always_inline)) T ExtractLane(const Vec128<T, N> v) {
static_assert(kLane < N, "Lane index out of bounds");
return static_cast<T>(_mm_extract_epi16(v.raw, kLane) & 0xFFFF);
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) T ExtractLane(const Vec128<T, N> v) {
static_assert(kLane < N, "Lane index out of bounds");
alignas(16) T lanes[4];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[kLane];
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) T ExtractLane(const Vec128<T, N> v) {
static_assert(kLane < N, "Lane index out of bounds");
alignas(16) T lanes[2];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[kLane];
}
template <size_t kLane, size_t N>
inline __attribute__((always_inline)) float ExtractLane(const Vec128<float, N> v) {
static_assert(kLane < N, "Lane index out of bounds");
alignas(16) float lanes[4];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[kLane];
}
template <size_t kLane>
inline __attribute__((always_inline)) double ExtractLane(const Vec128<double, 1> v) {
static_assert(kLane == 0, "Lane index out of bounds");
return GetLane(v);
}
template <size_t kLane>
inline __attribute__((always_inline)) double ExtractLane(const Vec128<double> v) {
static_assert(kLane < 2, "Lane index out of bounds");
const Half<DFromV<decltype(v)>> dh;
return kLane == 0 ? GetLane(v) : GetLane(UpperHalf(dh, v));
}
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T ExtractLane(const Vec128<T, 1> v, size_t i) {
do { } while (0);
(void)i;
return GetLane(v);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T ExtractLane(const Vec128<T, 2> v, size_t i) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::ExtractLane<0>(v);
case 1:
return detail::ExtractLane<1>(v);
}
}
alignas(16) T lanes[2];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[i];
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T ExtractLane(const Vec128<T, 4> v, size_t i) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::ExtractLane<0>(v);
case 1:
return detail::ExtractLane<1>(v);
case 2:
return detail::ExtractLane<2>(v);
case 3:
return detail::ExtractLane<3>(v);
}
}
alignas(16) T lanes[4];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[i];
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T ExtractLane(const Vec128<T, 8> v, size_t i) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::ExtractLane<0>(v);
case 1:
return detail::ExtractLane<1>(v);
case 2:
return detail::ExtractLane<2>(v);
case 3:
return detail::ExtractLane<3>(v);
case 4:
return detail::ExtractLane<4>(v);
case 5:
return detail::ExtractLane<5>(v);
case 6:
return detail::ExtractLane<6>(v);
case 7:
return detail::ExtractLane<7>(v);
}
}
alignas(16) T lanes[8];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[i];
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T ExtractLane(const Vec128<T, 16> v, size_t i) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::ExtractLane<0>(v);
case 1:
return detail::ExtractLane<1>(v);
case 2:
return detail::ExtractLane<2>(v);
case 3:
return detail::ExtractLane<3>(v);
case 4:
return detail::ExtractLane<4>(v);
case 5:
return detail::ExtractLane<5>(v);
case 6:
return detail::ExtractLane<6>(v);
case 7:
return detail::ExtractLane<7>(v);
case 8:
return detail::ExtractLane<8>(v);
case 9:
return detail::ExtractLane<9>(v);
case 10:
return detail::ExtractLane<10>(v);
case 11:
return detail::ExtractLane<11>(v);
case 12:
return detail::ExtractLane<12>(v);
case 13:
return detail::ExtractLane<13>(v);
case 14:
return detail::ExtractLane<14>(v);
case 15:
return detail::ExtractLane<15>(v);
}
}
alignas(16) T lanes[16];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[i];
}
namespace detail {
template <class V>
inline __attribute__((always_inline)) V InsertLaneUsingBroadcastAndBlend(V v, size_t i, TFromV<V> t) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
const auto mask = RebindMask(d, Iota(du, 0) == Set(du, static_cast<TU>(i)));
return IfThenElse(mask, Set(d, t), v);
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> InsertLane(const Vec128<T, N> v, T t) {
static_assert(kLane < N, "Lane index out of bounds");
return InsertLaneUsingBroadcastAndBlend(v, kLane, t);
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> InsertLane(const Vec128<T, N> v, T t) {
static_assert(kLane < N, "Lane index out of bounds");
return Vec128<T, N>{_mm_insert_epi16(v.raw, t, kLane)};
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> InsertLane(const Vec128<T, N> v, T t) {
static_assert(kLane < N, "Lane index out of bounds");
return InsertLaneUsingBroadcastAndBlend(v, kLane, t);
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> InsertLane(const Vec128<T, N> v, T t) {
static_assert(kLane < N, "Lane index out of bounds");
const DFromV<decltype(v)> d;
const RebindToFloat<decltype(d)> df;
const auto vt = BitCast(df, Set(d, t));
if (kLane == 0) {
return BitCast(
d, Vec128<double, N>{_mm_shuffle_pd(vt.raw, BitCast(df, v).raw, 2)});
}
return BitCast(
d, Vec128<double, N>{_mm_shuffle_pd(BitCast(df, v).raw, vt.raw, 0)});
}
template <size_t kLane, size_t N>
inline __attribute__((always_inline)) Vec128<float, N> InsertLane(const Vec128<float, N> v, float t) {
static_assert(kLane < N, "Lane index out of bounds");
return InsertLaneUsingBroadcastAndBlend(v, kLane, t);
}
template <size_t kLane>
inline __attribute__((always_inline)) Vec128<double, 1> InsertLane(const Vec128<double, 1> v, double t) {
static_assert(kLane == 0, "Lane index out of bounds");
return Set(DFromV<decltype(v)>(), t);
}
template <size_t kLane>
inline __attribute__((always_inline)) Vec128<double> InsertLane(const Vec128<double> v, double t) {
static_assert(kLane < 2, "Lane index out of bounds");
const DFromV<decltype(v)> d;
const Vec128<double> vt = Set(d, t);
if (kLane == 0) {
return Vec128<double>{_mm_shuffle_pd(vt.raw, v.raw, 2)};
}
return Vec128<double>{_mm_shuffle_pd(v.raw, vt.raw, 0)};
}
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> InsertLane(const Vec128<T, 1> v, size_t i, T t) {
do { } while (0);
(void)i;
return Set(DFromV<decltype(v)>(), t);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 2> InsertLane(const Vec128<T, 2> v, size_t i, T t) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::InsertLane<0>(v, t);
case 1:
return detail::InsertLane<1>(v, t);
}
}
return detail::InsertLaneUsingBroadcastAndBlend(v, i, t);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 4> InsertLane(const Vec128<T, 4> v, size_t i, T t) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::InsertLane<0>(v, t);
case 1:
return detail::InsertLane<1>(v, t);
case 2:
return detail::InsertLane<2>(v, t);
case 3:
return detail::InsertLane<3>(v, t);
}
}
return detail::InsertLaneUsingBroadcastAndBlend(v, i, t);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 8> InsertLane(const Vec128<T, 8> v, size_t i, T t) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::InsertLane<0>(v, t);
case 1:
return detail::InsertLane<1>(v, t);
case 2:
return detail::InsertLane<2>(v, t);
case 3:
return detail::InsertLane<3>(v, t);
case 4:
return detail::InsertLane<4>(v, t);
case 5:
return detail::InsertLane<5>(v, t);
case 6:
return detail::InsertLane<6>(v, t);
case 7:
return detail::InsertLane<7>(v, t);
}
}
return detail::InsertLaneUsingBroadcastAndBlend(v, i, t);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 16> InsertLane(const Vec128<T, 16> v, size_t i, T t) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::InsertLane<0>(v, t);
case 1:
return detail::InsertLane<1>(v, t);
case 2:
return detail::InsertLane<2>(v, t);
case 3:
return detail::InsertLane<3>(v, t);
case 4:
return detail::InsertLane<4>(v, t);
case 5:
return detail::InsertLane<5>(v, t);
case 6:
return detail::InsertLane<6>(v, t);
case 7:
return detail::InsertLane<7>(v, t);
case 8:
return detail::InsertLane<8>(v, t);
case 9:
return detail::InsertLane<9>(v, t);
case 10:
return detail::InsertLane<10>(v, t);
case 11:
return detail::InsertLane<11>(v, t);
case 12:
return detail::InsertLane<12>(v, t);
case 13:
return detail::InsertLane<13>(v, t);
case 14:
return detail::InsertLane<14>(v, t);
case 15:
return detail::InsertLane<15>(v, t);
}
}
return detail::InsertLaneUsingBroadcastAndBlend(v, i, t);
}
template <int kBytes, class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> CombineShiftRightBytes(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, Vec128<uint8_t>{_mm_alignr_epi8(
BitCast(d8, hi).raw, BitCast(d8, lo).raw, kBytes)});
}
template <int kBytes, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> CombineShiftRightBytes(D d, VFromD<D> hi, VFromD<D> lo) {
constexpr size_t kSize = d.MaxBytes();
static_assert(0 < kBytes && kBytes < kSize, "kBytes invalid");
const Repartition<uint8_t, decltype(d)> d8;
using V8 = Vec128<uint8_t>;
const DFromV<V8> dfull8;
const Repartition<TFromD<D>, decltype(dfull8)> dfull;
const V8 hi8{BitCast(d8, hi).raw};
const V8 lo8 = ShiftLeftBytes<16 - kSize>(V8{BitCast(d8, lo).raw});
const V8 r = CombineShiftRightBytes<16 - kSize + kBytes>(dfull8, hi8, lo8);
return VFromD<D>{BitCast(dfull, r).raw};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> Broadcast(const Vec128<uint16_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
if (kLane < 4) {
const __m128i lo = _mm_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF);
return Vec128<uint16_t, N>{_mm_unpacklo_epi64(lo, lo)};
} else {
const __m128i hi = _mm_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF);
return Vec128<uint16_t, N>{_mm_unpackhi_epi64(hi, hi)};
}
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> Broadcast(const Vec128<uint32_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<uint32_t, N>{_mm_shuffle_epi32(v.raw, 0x55 * kLane)};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> Broadcast(const Vec128<uint64_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<uint64_t, N>{_mm_shuffle_epi32(v.raw, kLane ? 0xEE : 0x44)};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> Broadcast(const Vec128<int16_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
if (kLane < 4) {
const __m128i lo = _mm_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF);
return Vec128<int16_t, N>{_mm_unpacklo_epi64(lo, lo)};
} else {
const __m128i hi = _mm_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF);
return Vec128<int16_t, N>{_mm_unpackhi_epi64(hi, hi)};
}
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> Broadcast(const Vec128<int32_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<int32_t, N>{_mm_shuffle_epi32(v.raw, 0x55 * kLane)};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> Broadcast(const Vec128<int64_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<int64_t, N>{_mm_shuffle_epi32(v.raw, kLane ? 0xEE : 0x44)};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Broadcast(const Vec128<float, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<float, N>{_mm_shuffle_ps(v.raw, v.raw, 0x55 * kLane)};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Broadcast(const Vec128<double, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<double, N>{_mm_shuffle_pd(v.raw, v.raw, 3 * kLane)};
}
template <typename T, size_t N = 16 / sizeof(T)>
struct Indices128 {
__m128i raw;
};
template <class D, typename T = TFromD<D>, typename TI, size_t kN,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Indices128<T, kN> IndicesFromVec(D d, Vec128<TI, kN> vec) {
static_assert(sizeof(T) == sizeof(TI), "Index size must match lane");
(void)d;
return Indices128<T, kN>{vec.raw};
}
template <class D, typename T = TFromD<D>, typename TI, size_t kN,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Indices128<T, kN> IndicesFromVec(D d, Vec128<TI, kN> vec) {
static_assert(sizeof(T) == sizeof(TI), "Index size must match lane");
const Repartition<uint8_t, decltype(d)> d8;
using V8 = VFromD<decltype(d8)>;
alignas(16) static constexpr uint8_t kByteOffsets[16] = {
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1};
alignas(16) static constexpr uint8_t kBroadcastLaneBytes[16] = {
0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14};
const V8 lane_indices = TableLookupBytes(vec, Load(d8, kBroadcastLaneBytes));
const Repartition<uint16_t, decltype(d)> d16;
const V8 byte_indices = BitCast(d8, ShiftLeft<1>(BitCast(d16, lane_indices)));
return Indices128<T, kN>{Add(byte_indices, Load(d8, kByteOffsets)).raw};
}
template <class D, typename T = TFromD<D>, typename TI, size_t kN,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Indices128<T, kN> IndicesFromVec(D d, Vec128<TI, kN> vec) {
static_assert(sizeof(T) == sizeof(TI), "Index size must match lane");
const Repartition<uint8_t, decltype(d)> d8;
using V8 = VFromD<decltype(d8)>;
alignas(16) static constexpr uint8_t kByteOffsets[16] = {
0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3};
alignas(16) static constexpr uint8_t kBroadcastLaneBytes[16] = {
0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12};
const V8 lane_indices = TableLookupBytes(vec, Load(d8, kBroadcastLaneBytes));
const Repartition<uint16_t, decltype(d)> d16;
const V8 byte_indices = BitCast(d8, ShiftLeft<2>(BitCast(d16, lane_indices)));
return Indices128<T, kN>{Add(byte_indices, Load(d8, kByteOffsets)).raw};
}
template <class D, typename T = TFromD<D>, typename TI, size_t kN,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Indices128<T, kN> IndicesFromVec(D d, Vec128<TI, kN> vec) {
static_assert(sizeof(T) == sizeof(TI), "Index size must match lane");
(void)d;
return Indices128<T, kN>{vec.raw};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename TI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Indices128<TFromD<D>, D::kPrivateLanes> SetTableIndices(
D d, const TI* idx) {
static_assert(sizeof(TFromD<D>) == sizeof(TI), "Index size must match lane");
const Rebind<TI, decltype(d)> di;
return IndicesFromVec(d, LoadU(di, idx));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> TableLookupLanes(Vec128<T, N> v, Indices128<T, N> idx) {
return TableLookupBytes(v, Vec128<T, N>{idx.raw});
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> TableLookupLanes(Vec128<T, N> v, Indices128<T, N> idx) {
return TableLookupBytes(v, Vec128<T, N>{idx.raw});
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> TableLookupLanes(Vec128<T, N> v, Indices128<T, N> idx) {
return TableLookupBytes(v, Vec128<T, N>{idx.raw});
}
template <size_t N, hwy::EnableIf<(N * sizeof(float) > 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> TableLookupLanes(Vec128<float, N> v,
Indices128<float, N> idx) {
const DFromV<decltype(v)> df;
const RebindToSigned<decltype(df)> di;
return BitCast(df,
TableLookupBytes(BitCast(di, v), Vec128<int32_t, N>{idx.raw}));
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> TableLookupLanes(Vec128<T, 1> v,
Indices128<T, 1> ) {
return v;
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> TableLookupLanes(Vec128<T> v, Indices128<T> idx) {
const DFromV<decltype(v)> d;
Vec128<int64_t> vidx{idx.raw};
const RebindToSigned<decltype(d)> di;
const Vec128<int64_t> same = (vidx ^ Iota(di, 0)) - Set(di, 1);
const Mask128<T> mask_same = RebindMask(d, MaskFromVec(same));
return IfThenElse(mask_same, v, Shuffle01(v));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> TableLookupLanes(Vec128<double> v,
Indices128<double> idx) {
Vec128<int64_t> vidx{idx.raw};
const DFromV<decltype(v)> d;
const RebindToSigned<decltype(d)> di;
const Vec128<int64_t> same = (vidx ^ Iota(di, 0)) - Set(di, 1);
const Mask128<double> mask_same = RebindMask(d, MaskFromVec(same));
return IfThenElse(mask_same, v, Shuffle01(v));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReverseBlocks(D , VFromD<D> v) {
return v;
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> Reverse(D , Vec128<T, 1> v) {
return v;
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> Reverse(D , const Vec64<T> v) {
return Vec64<T>{Shuffle2301(Vec128<T>{v.raw}).raw};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Reverse(D , const Vec128<T> v) {
return Shuffle01(v);
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Reverse(D , const Vec128<T> v) {
return Shuffle0123(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse(D d, const VFromD<D> v) {
constexpr size_t kN = MaxLanes(d);
if (kN == 1) return v;
if (kN == 2) {
return VFromD<D>{_mm_shufflelo_epi16(v.raw,
(((
0
) << 6) | ((
1
) << 4) | ((
0
) << 2) | (
1
))
)};
}
if (kN == 4) {
return VFromD<D>{_mm_shufflelo_epi16(v.raw,
(((
0
) << 6) | ((
1
) << 4) | ((
2
) << 2) | (
3
))
)};
}
const RebindToSigned<decltype(d)> di;
alignas(16) static constexpr int16_t kShuffle[8] = {
0x0F0E, 0x0D0C, 0x0B0A, 0x0908, 0x0706, 0x0504, 0x0302, 0x0100};
return BitCast(d, TableLookupBytes(v, LoadDup128(di, kShuffle)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse(D d, const VFromD<D> v) {
constexpr int kN = static_cast<int>(MaxLanes(d));
if (kN == 1) return v;
alignas(16) static constexpr int8_t kReverse[16] = {
kN - 1, kN - 2, kN - 3, kN - 4, kN - 5, kN - 6, kN - 7, kN - 8,
kN - 9, kN - 10, kN - 11, kN - 12, kN - 13, kN - 14, kN - 15, kN - 16};
const RebindToSigned<decltype(d)> di;
const VFromD<decltype(di)> idx = Load(di, kReverse);
return VFromD<D>{_mm_shuffle_epi8(BitCast(di, v).raw, idx.raw)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> Reverse2(D , Vec128<T, 1> v) {
return v;
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse2(D d, VFromD<D> v) {
const RebindToSigned<decltype(d)> di;
alignas(16) static constexpr int16_t kShuffle[8] = {
0x0302, 0x0100, 0x0706, 0x0504, 0x0B0A, 0x0908, 0x0F0E, 0x0D0C};
return BitCast(d, TableLookupBytes(v, LoadDup128(di, kShuffle)));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse2(D , VFromD<D> v) {
return Shuffle2301(v);
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse2(D , VFromD<D> v) {
return Shuffle01(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse4(D d, VFromD<D> v) {
constexpr size_t kN = MaxLanes(d);
if (kN <= 4) {
return VFromD<D>{_mm_shufflelo_epi16(v.raw,
(((
0
) << 6) | ((
1
) << 4) | ((
2
) << 2) | (
3
))
)};
}
const RebindToSigned<decltype(d)> di;
alignas(16) static constexpr int16_t kShuffle[8] = {
0x0706, 0x0504, 0x0302, 0x0100, 0x0F0E, 0x0D0C, 0x0B0A, 0x0908};
return BitCast(d, TableLookupBytes(v, LoadDup128(di, kShuffle)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse4(D , const VFromD<D> v) {
return Shuffle0123(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse4(D , VFromD<D> ) {
do { if (!(0)) { ::hwy::Abort("/home/mathieu/Perso/highway/hwy/ops/x86_128-inl.h", 4907, "Assert %s","0"); } } while (0);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse8(D d, const VFromD<D> v) {
const RebindToSigned<decltype(d)> di;
alignas(16) static constexpr int16_t kShuffle[8] = {
0x0F0E, 0x0D0C, 0x0B0A, 0x0908, 0x0706, 0x0504, 0x0302, 0x0100};
return BitCast(d, TableLookupBytes(v, LoadDup128(di, kShuffle)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr,
hwy::EnableIf<((size_t{1} << sizeof(TFromD<D>)) & ((1 << 4) | (1 << 8))) != 0>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse8(D , VFromD<D> ) {
do { if (!(0)) { ::hwy::Abort("/home/mathieu/Perso/highway/hwy/ops/x86_128-inl.h", 4928, "Assert %s","0"); } } while (0);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> InterleaveLower(Vec128<uint8_t, N> a,
Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_unpacklo_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> InterleaveLower(Vec128<uint16_t, N> a,
Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_unpacklo_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> InterleaveLower(Vec128<uint32_t, N> a,
Vec128<uint32_t, N> b) {
return Vec128<uint32_t, N>{_mm_unpacklo_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> InterleaveLower(Vec128<uint64_t, N> a,
Vec128<uint64_t, N> b) {
return Vec128<uint64_t, N>{_mm_unpacklo_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> InterleaveLower(Vec128<int8_t, N> a,
Vec128<int8_t, N> b) {
return Vec128<int8_t, N>{_mm_unpacklo_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> InterleaveLower(Vec128<int16_t, N> a,
Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_unpacklo_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> InterleaveLower(Vec128<int32_t, N> a,
Vec128<int32_t, N> b) {
return Vec128<int32_t, N>{_mm_unpacklo_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> InterleaveLower(Vec128<int64_t, N> a,
Vec128<int64_t, N> b) {
return Vec128<int64_t, N>{_mm_unpacklo_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> InterleaveLower(Vec128<float, N> a,
Vec128<float, N> b) {
return Vec128<float, N>{_mm_unpacklo_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> InterleaveLower(Vec128<double, N> a,
Vec128<double, N> b) {
return Vec128<double, N>{_mm_unpacklo_pd(a.raw, b.raw)};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> InterleaveLower(D , VFromD<D> a, VFromD<D> b) {
return InterleaveLower(a, b);
}
namespace detail {
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t> InterleaveUpper(Vec128<uint8_t> a, Vec128<uint8_t> b) {
return Vec128<uint8_t>{_mm_unpackhi_epi8(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t> InterleaveUpper(Vec128<uint16_t> a,
Vec128<uint16_t> b) {
return Vec128<uint16_t>{_mm_unpackhi_epi16(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> InterleaveUpper(Vec128<uint32_t> a,
Vec128<uint32_t> b) {
return Vec128<uint32_t>{_mm_unpackhi_epi32(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t> InterleaveUpper(Vec128<uint64_t> a,
Vec128<uint64_t> b) {
return Vec128<uint64_t>{_mm_unpackhi_epi64(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t> InterleaveUpper(Vec128<int8_t> a, Vec128<int8_t> b) {
return Vec128<int8_t>{_mm_unpackhi_epi8(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t> InterleaveUpper(Vec128<int16_t> a, Vec128<int16_t> b) {
return Vec128<int16_t>{_mm_unpackhi_epi16(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> InterleaveUpper(Vec128<int32_t> a, Vec128<int32_t> b) {
return Vec128<int32_t>{_mm_unpackhi_epi32(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t> InterleaveUpper(Vec128<int64_t> a, Vec128<int64_t> b) {
return Vec128<int64_t>{_mm_unpackhi_epi64(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> InterleaveUpper(Vec128<float> a, Vec128<float> b) {
return Vec128<float>{_mm_unpackhi_ps(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> InterleaveUpper(Vec128<double> a, Vec128<double> b) {
return Vec128<double>{_mm_unpackhi_pd(a.raw, b.raw)};
}
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> InterleaveUpper(D , Vec128<T> a, Vec128<T> b) {
return detail::InterleaveUpper(a, b);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> InterleaveUpper(D d, VFromD<D> a, VFromD<D> b) {
const Half<decltype(d)> d2;
return InterleaveLower(d, VFromD<D>{UpperHalf(d2, a).raw},
VFromD<D>{UpperHalf(d2, b).raw});
}
template <int kLane, class T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Broadcast(const Vec128<T, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
const DFromV<decltype(v)> d;
return TableLookupBytes(v, Set(d, static_cast<T>(kLane)));
}
template <class V, class DW = RepartitionToWide<DFromV<V>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DW> ZipLower(V a, V b) {
return BitCast(DW(), InterleaveLower(a, b));
}
template <class V, class D = DFromV<V>, class DW = RepartitionToWide<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DW> ZipLower(DW dw, V a, V b) {
return BitCast(dw, InterleaveLower(D(), a, b));
}
template <class V, class D = DFromV<V>, class DW = RepartitionToWide<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DW> ZipUpper(DW dw, V a, V b) {
return BitCast(dw, InterleaveUpper(D(), a, b));
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Per4LaneBlkShufDupSet4xU32(D d, const uint32_t x3,
const uint32_t x2,
const uint32_t x1,
const uint32_t x0) {
return ResizeBitCast(
d, Vec128<uint32_t>{_mm_set_epi32(
static_cast<int32_t>(x3), static_cast<int32_t>(x2),
static_cast<int32_t>(x1), static_cast<int32_t>(x0))});
}
template <size_t kIdx3210, class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> ,
hwy::SizeTag<2> ,
hwy::SizeTag<8> , V v) {
return V{_mm_shufflelo_epi16(v.raw, static_cast<int>(kIdx3210 & 0xFF))};
}
template <size_t kIdx3210, class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >()>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> ,
hwy::SizeTag<4> ,
hwy::SizeTag<16> , V v) {
return V{_mm_shuffle_epi32(v.raw, static_cast<int>(kIdx3210 & 0xFF))};
}
template <size_t kIdx3210, class V, hwy::EnableIf<hwy::IsFloat<TFromV<V> >()>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> ,
hwy::SizeTag<4> ,
hwy::SizeTag<16> , V v) {
return V{_mm_shuffle_ps(v.raw, v.raw, static_cast<int>(kIdx3210 & 0xFF))};
}
}
namespace detail {
template <class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 8>* = nullptr>
inline __attribute__((always_inline)) V SlideUpLanes(V v, size_t amt) {
const DFromV<decltype(v)> d;
const Full64<uint64_t> du64;
const auto vu64 = ResizeBitCast(du64, v);
return ResizeBitCast(
d, ShiftLeftSame(vu64, static_cast<int>(amt * sizeof(TFromV<V>) * 8)));
}
template <class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) == 16>* = nullptr>
inline __attribute__((always_inline)) V SlideUpLanes(V v, size_t amt) {
const DFromV<decltype(v)> d;
const Repartition<uint8_t, decltype(d)> du8;
const auto idx =
Iota(du8, static_cast<uint8_t>(size_t{0} - amt * sizeof(TFromV<V>)));
return BitCast(d, TableLookupBytesOr0(BitCast(du8, v), idx));
}
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpLanes(D , VFromD<D> v, size_t ) {
return v;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftLeftLanes<1>(d, v);
}
}
return detail::SlideUpLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftLeftLanes<1>(d, v);
case 2:
return ShiftLeftLanes<2>(d, v);
case 3:
return ShiftLeftLanes<3>(d, v);
}
}
return detail::SlideUpLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftLeftLanes<1>(d, v);
case 2:
return ShiftLeftLanes<2>(d, v);
case 3:
return ShiftLeftLanes<3>(d, v);
case 4:
return ShiftLeftLanes<4>(d, v);
case 5:
return ShiftLeftLanes<5>(d, v);
case 6:
return ShiftLeftLanes<6>(d, v);
case 7:
return ShiftLeftLanes<7>(d, v);
}
}
return detail::SlideUpLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 16)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftLeftLanes<1>(d, v);
case 2:
return ShiftLeftLanes<2>(d, v);
case 3:
return ShiftLeftLanes<3>(d, v);
case 4:
return ShiftLeftLanes<4>(d, v);
case 5:
return ShiftLeftLanes<5>(d, v);
case 6:
return ShiftLeftLanes<6>(d, v);
case 7:
return ShiftLeftLanes<7>(d, v);
case 8:
return ShiftLeftLanes<8>(d, v);
case 9:
return ShiftLeftLanes<9>(d, v);
case 10:
return ShiftLeftLanes<10>(d, v);
case 11:
return ShiftLeftLanes<11>(d, v);
case 12:
return ShiftLeftLanes<12>(d, v);
case 13:
return ShiftLeftLanes<13>(d, v);
case 14:
return ShiftLeftLanes<14>(d, v);
case 15:
return ShiftLeftLanes<15>(d, v);
}
}
return detail::SlideUpLanes(v, amt);
}
namespace detail {
template <class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 8>* = nullptr>
inline __attribute__((always_inline)) V SlideDownLanes(V v, size_t amt) {
const DFromV<decltype(v)> d;
const Repartition<UnsignedFromSize<d.MaxBytes()>, decltype(d)> dv;
return BitCast(d,
ShiftRightSame(BitCast(dv, v),
static_cast<int>(amt * sizeof(TFromV<V>) * 8)));
}
template <class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) == 16>* = nullptr>
inline __attribute__((always_inline)) V SlideDownLanes(V v, size_t amt) {
const DFromV<decltype(v)> d;
const Repartition<int8_t, decltype(d)> di8;
auto idx = Iota(di8, static_cast<int8_t>(amt * sizeof(TFromV<V>)));
idx = Or(idx, VecFromMask(di8, idx > Set(di8, int8_t{15})));
return BitCast(d, TableLookupBytesOr0(BitCast(di8, v), idx));
}
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownLanes(D , VFromD<D> v, size_t ) {
return v;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftRightLanes<1>(d, v);
}
}
return detail::SlideDownLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftRightLanes<1>(d, v);
case 2:
return ShiftRightLanes<2>(d, v);
case 3:
return ShiftRightLanes<3>(d, v);
}
}
return detail::SlideDownLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftRightLanes<1>(d, v);
case 2:
return ShiftRightLanes<2>(d, v);
case 3:
return ShiftRightLanes<3>(d, v);
case 4:
return ShiftRightLanes<4>(d, v);
case 5:
return ShiftRightLanes<5>(d, v);
case 6:
return ShiftRightLanes<6>(d, v);
case 7:
return ShiftRightLanes<7>(d, v);
}
}
return detail::SlideDownLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 16)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftRightLanes<1>(d, v);
case 2:
return ShiftRightLanes<2>(d, v);
case 3:
return ShiftRightLanes<3>(d, v);
case 4:
return ShiftRightLanes<4>(d, v);
case 5:
return ShiftRightLanes<5>(d, v);
case 6:
return ShiftRightLanes<6>(d, v);
case 7:
return ShiftRightLanes<7>(d, v);
case 8:
return ShiftRightLanes<8>(d, v);
case 9:
return ShiftRightLanes<9>(d, v);
case 10:
return ShiftRightLanes<10>(d, v);
case 11:
return ShiftRightLanes<11>(d, v);
case 12:
return ShiftRightLanes<12>(d, v);
case 13:
return ShiftRightLanes<13>(d, v);
case 14:
return ShiftRightLanes<14>(d, v);
case 15:
return ShiftRightLanes<15>(d, v);
}
}
return detail::SlideDownLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, class VH = VFromD<Half<D>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Combine(D d, VH hi_half, VH lo_half) {
const Half<decltype(d)> dh;
const RebindToUnsigned<decltype(dh)> duh;
using VU = Vec128<UnsignedFromSize<dh.MaxBytes()>, 2>;
const VU lo{BitCast(duh, lo_half).raw};
const VU hi{BitCast(duh, hi_half).raw};
return BitCast(d, InterleaveLower(lo, hi));
}
namespace detail {
template <class D, typename T = TFromD<D>>
inline __attribute__((always_inline)) Vec128<T> ZeroExtendVector(hwy::NonFloatTag , D ,
Vec64<T> lo) {
return Vec128<T>{_mm_move_epi64(lo.raw)};
}
template <class D, typename T = TFromD<D>>
inline __attribute__((always_inline)) Vec128<T> ZeroExtendVector(hwy::FloatTag , D d, Vec64<T> lo) {
const RebindToUnsigned<decltype(d)> du;
return BitCast(d, ZeroExtendVector(du, BitCast(Half<decltype(du)>(), lo)));
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ZeroExtendVector(D d, Vec64<T> lo) {
return detail::ZeroExtendVector(hwy::IsFloatTag<T>(), d, lo);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ZeroExtendVector(D d, VFromD<Half<D>> lo) {
const Half<D> dh;
return IfThenElseZero(FirstN(d, MaxLanes(dh)), VFromD<D>{lo.raw});
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatLowerLower(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<uint64_t, decltype(d)> d64;
return BitCast(d, InterleaveLower(BitCast(d64, lo), BitCast(d64, hi)));
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatUpperUpper(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<uint64_t, decltype(d)> d64;
return BitCast(d, InterleaveUpper(d64, BitCast(d64, lo), BitCast(d64, hi)));
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatLowerUpper(D d, Vec128<T> hi, Vec128<T> lo) {
return CombineShiftRightBytes<8>(d, hi, lo);
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatUpperLower(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<double, decltype(d)> dd;
return BitCast(
d, Vec128<double>{_mm_shuffle_pd(BitCast(dd, lo).raw, BitCast(dd, hi).raw,
(((
1
) << 1) | (
0
))
)});
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> ConcatUpperLower(D d, Vec128<float> hi,
Vec128<float> lo) {
(void)d;
return Vec128<float>{_mm_shuffle_ps(lo.raw, hi.raw,
(((
3
) << 6) | ((
2
) << 4) | ((
1
) << 2) | (
0
))
)};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> ConcatUpperLower(D , Vec128<double> hi,
Vec128<double> lo) {
return Vec128<double>{_mm_shuffle_pd(lo.raw, hi.raw,
(((
1
) << 1) | (
0
))
)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatLowerLower(D d, VFromD<D> hi, VFromD<D> lo) {
const Half<decltype(d)> d2;
return Combine(d, LowerHalf(d2, hi), LowerHalf(d2, lo));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatUpperUpper(D d, VFromD<D> hi, VFromD<D> lo) {
const Half<decltype(d)> d2;
return Combine(d, UpperHalf(d2, hi), UpperHalf(d2, lo));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatLowerUpper(D d, const VFromD<D> hi,
const VFromD<D> lo) {
const Half<decltype(d)> d2;
return Combine(d, LowerHalf(d2, hi), UpperHalf(d2, lo));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatUpperLower(D d, VFromD<D> hi, VFromD<D> lo) {
const Half<decltype(d)> d2;
return Combine(d, UpperHalf(d2, hi), LowerHalf(d2, lo));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatOdd(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<uint16_t, decltype(d)> dw;
const Vec128<uint16_t> uH = ShiftRight<8>(BitCast(dw, hi));
const Vec128<uint16_t> uL = ShiftRight<8>(BitCast(dw, lo));
return Vec128<T>{_mm_packus_epi16(uL.raw, uH.raw)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ConcatOdd(D d, Vec64<T> hi, Vec64<T> lo) {
const Repartition<uint32_t, decltype(d)> du32;
alignas(16) const uint8_t kCompactOddU8[8] = {1, 3, 5, 7};
const Vec64<T> shuf = BitCast(d, Load(Full64<uint8_t>(), kCompactOddU8));
const Vec64<T> L = TableLookupBytes(lo, shuf);
const Vec64<T> H = TableLookupBytes(hi, shuf);
return BitCast(d, InterleaveLower(du32, BitCast(du32, L), BitCast(du32, H)));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<T> ConcatOdd(D d, Vec32<T> hi, Vec32<T> lo) {
const Repartition<uint16_t, decltype(d)> du16;
alignas(16) const uint8_t kCompactOddU8[4] = {1, 3};
const Vec32<T> shuf = BitCast(d, Load(Full32<uint8_t>(), kCompactOddU8));
const Vec32<T> L = TableLookupBytes(lo, shuf);
const Vec32<T> H = TableLookupBytes(hi, shuf);
return BitCast(d, InterleaveLower(du16, BitCast(du16, L), BitCast(du16, H)));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatOdd(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<int32_t, decltype(d)> dw;
const Vec128<int32_t> uH = ShiftRight<16>(BitCast(dw, hi));
const Vec128<int32_t> uL = ShiftRight<16>(BitCast(dw, lo));
return Vec128<T>{_mm_packs_epi32(uL.raw, uH.raw)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ConcatOdd(D d, Vec64<T> hi, Vec64<T> lo) {
const Repartition<uint32_t, decltype(d)> du32;
alignas(16) const uint8_t kCompactOddU16[8] = {2, 3, 6, 7};
const Vec64<T> shuf = BitCast(d, Load(Full64<uint8_t>(), kCompactOddU16));
const Vec64<T> L = TableLookupBytes(lo, shuf);
const Vec64<T> H = TableLookupBytes(hi, shuf);
return BitCast(d, InterleaveLower(du32, BitCast(du32, L), BitCast(du32, H)));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatOdd(D d, Vec128<T> hi, Vec128<T> lo) {
const RebindToFloat<decltype(d)> df;
return BitCast(
d, Vec128<float>{_mm_shuffle_ps(BitCast(df, lo).raw, BitCast(df, hi).raw,
(((
3
) << 6) | ((
1
) << 4) | ((
3
) << 2) | (
1
))
)});
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> ConcatOdd(D , Vec128<float> hi, Vec128<float> lo) {
return Vec128<float>{_mm_shuffle_ps(lo.raw, hi.raw,
(((
3
) << 6) | ((
1
) << 4) | ((
3
) << 2) | (
1
))
)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 2> ConcatOdd(D d, Vec128<T, 2> hi, Vec128<T, 2> lo) {
return InterleaveUpper(d, lo, hi);
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatEven(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<uint16_t, decltype(d)> dw;
const Vec128<uint16_t> mask = Set(dw, 0x00FF);
const Vec128<uint16_t> uH = And(BitCast(dw, hi), mask);
const Vec128<uint16_t> uL = And(BitCast(dw, lo), mask);
return Vec128<T>{_mm_packus_epi16(uL.raw, uH.raw)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ConcatEven(D d, Vec64<T> hi, Vec64<T> lo) {
const Repartition<uint32_t, decltype(d)> du32;
alignas(16) const uint8_t kCompactEvenU8[8] = {0, 2, 4, 6};
const Vec64<T> shuf = BitCast(d, Load(Full64<uint8_t>(), kCompactEvenU8));
const Vec64<T> L = TableLookupBytes(lo, shuf);
const Vec64<T> H = TableLookupBytes(hi, shuf);
return BitCast(d, InterleaveLower(du32, BitCast(du32, L), BitCast(du32, H)));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<T> ConcatEven(D d, Vec32<T> hi, Vec32<T> lo) {
const Repartition<uint16_t, decltype(d)> du16;
alignas(16) const uint8_t kCompactEvenU8[4] = {0, 2};
const Vec32<T> shuf = BitCast(d, Load(Full32<uint8_t>(), kCompactEvenU8));
const Vec32<T> L = TableLookupBytes(lo, shuf);
const Vec32<T> H = TableLookupBytes(hi, shuf);
return BitCast(d, InterleaveLower(du16, BitCast(du16, L), BitCast(du16, H)));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatEven(D d, Vec128<T> hi, Vec128<T> lo) {
alignas(16) const T kCompactEvenU16[8] = {0x0100, 0x0504, 0x0908, 0x0D0C};
const Vec128<T> shuf = BitCast(d, Load(d, kCompactEvenU16));
const Vec128<T> L = TableLookupBytes(lo, shuf);
const Vec128<T> H = TableLookupBytes(hi, shuf);
return ConcatLowerLower(d, H, L);
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ConcatEven(D d, Vec64<T> hi, Vec64<T> lo) {
const Repartition<uint32_t, decltype(d)> du32;
alignas(16) const uint8_t kCompactEvenU16[8] = {0, 1, 4, 5};
const Vec64<T> shuf = BitCast(d, Load(Full64<uint8_t>(), kCompactEvenU16));
const Vec64<T> L = TableLookupBytes(lo, shuf);
const Vec64<T> H = TableLookupBytes(hi, shuf);
return BitCast(d, InterleaveLower(du32, BitCast(du32, L), BitCast(du32, H)));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatEven(D d, Vec128<T> hi, Vec128<T> lo) {
const RebindToFloat<decltype(d)> df;
return BitCast(
d, Vec128<float>{_mm_shuffle_ps(BitCast(df, lo).raw, BitCast(df, hi).raw,
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)});
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> ConcatEven(D , Vec128<float> hi,
Vec128<float> lo) {
return Vec128<float>{_mm_shuffle_ps(lo.raw, hi.raw,
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)};
}
template <typename D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 2> ConcatEven(D d, Vec128<T, 2> hi, Vec128<T, 2> lo) {
return InterleaveLower(d, lo, hi);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> DupEven(const Vec128<T, 1> v) {
return v;
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 2> DupEven(const Vec128<T, 2> v) {
return InterleaveLower(DFromV<decltype(v)>(), v, v);
}
template <typename V, hwy::EnableIf<sizeof(TFromV<V>) == (1)>* = nullptr, hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) > 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V DupEven(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint8_t kShuffle[16] = {
0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14};
return TableLookupBytes(v, BitCast(d, LoadDup128(du, kShuffle)));
}
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> DupEven(const Vec64<T> v) {
return Vec64<T>{_mm_shufflelo_epi16(v.raw,
(((
2
) << 6) | ((
2
) << 4) | ((
0
) << 2) | (
0
))
)};
}
template <typename V, hwy::EnableIf<sizeof(TFromV<V>) == (2)>* = nullptr, hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V DupEven(V v) {
alignas(16) static constexpr uint16_t kShuffle[8] = {
0x0100, 0x0100, 0x0504, 0x0504, 0x0908, 0x0908, 0x0d0c, 0x0d0c};
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
return TableLookupBytes(v, BitCast(d, LoadDup128(du, kShuffle)));
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> DupEven(Vec128<T> v) {
return Vec128<T>{_mm_shuffle_epi32(v.raw,
(((
2
) << 6) | ((
2
) << 4) | ((
0
) << 2) | (
0
))
)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> DupEven(Vec128<float> v) {
return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw,
(((
2
) << 6) | ((
2
) << 4) | ((
0
) << 2) | (
0
))
)};
}
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> DupOdd(Vec128<T, 1> v) {
return v;
}
template <typename V, hwy::EnableIf<sizeof(TFromV<V>) == (1)>* = nullptr, hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V DupOdd(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint8_t kShuffle[16] = {
1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15};
return TableLookupBytes(v, BitCast(d, LoadDup128(du, kShuffle)));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr, hwy::EnableIf<(N <= 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> DupOdd(Vec128<T, N> v) {
return Vec128<T, N>{_mm_shufflelo_epi16(v.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
}
template <typename V, hwy::EnableIf<sizeof(TFromV<V>) == (2)>* = nullptr, hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V DupOdd(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint16_t kShuffle[8] = {
0x0302, 0x0302, 0x0706, 0x0706, 0x0b0a, 0x0b0a, 0x0f0e, 0x0f0e};
return TableLookupBytes(v, BitCast(d, LoadDup128(du, kShuffle)));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> DupOdd(Vec128<T, N> v) {
return Vec128<T, N>{_mm_shuffle_epi32(v.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> DupOdd(Vec128<float, N> v) {
return Vec128<float, N>{
_mm_shuffle_ps(v.raw, v.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> DupOdd(const Vec128<T, N> v) {
return InterleaveUpper(DFromV<decltype(v)>(), v, v);
}
template <typename T, size_t N, hwy::EnableIf<N * sizeof(T) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> TwoTablesLookupLanes(Vec128<T, N> a, Vec128<T, N> b,
Indices128<T, N> idx) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
const Indices128<T, N * 2> idx2{idx.raw};
return LowerHalf(d, TableLookupLanes(Combine(dt, b, a), idx2));
}
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> TwoTablesLookupLanes(Vec128<T> a, Vec128<T> b,
Indices128<T> idx) {
const DFromV<decltype(a)> d;
const Vec128<T> idx_vec{idx.raw};
const RebindToSigned<decltype(d)> di;
const auto sel_hi_mask =
RebindMask(d, BitCast(di, idx_vec) > Set(di, int8_t{15}));
const auto lo_lookup_result = TableLookupBytes(a, idx_vec);
const auto hi_lookup_result = TableLookupBytes(b, idx_vec);
return IfThenElse(sel_hi_mask, hi_lookup_result, lo_lookup_result);
}
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> TwoTablesLookupLanes(Vec128<T> a, Vec128<T> b,
Indices128<T> idx) {
const DFromV<decltype(a)> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, TwoTablesLookupLanes(BitCast(du8, a), BitCast(du8, b),
Indices128<uint8_t>{idx.raw}));
}
template <typename T, hwy::EnableIf<IsSame<T, uint32_t>() || IsSame<T, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> TwoTablesLookupLanes(Vec128<T> a, Vec128<T> b,
Indices128<T> idx) {
const DFromV<decltype(a)> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, TwoTablesLookupLanes(BitCast(du8, a), BitCast(du8, b),
Indices128<uint8_t>{idx.raw}));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> TwoTablesLookupLanes(Vec128<float> a, Vec128<float> b,
Indices128<float> idx) {
const DFromV<decltype(a)> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, TwoTablesLookupLanes(BitCast(du8, a), BitCast(du8, b),
Indices128<uint8_t>{idx.raw}));
}
template <typename T, hwy::EnableIf<IsSame<T, uint64_t>() || IsSame<T, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> TwoTablesLookupLanes(Vec128<T> a, Vec128<T> b,
Indices128<T> idx) {
const DFromV<decltype(a)> d;
const Vec128<T> idx_vec{idx.raw};
const Indices128<T> idx_mod{And(idx_vec, Set(d, T{1})).raw};
const Repartition<int32_t, decltype(d)> di32;
const RebindToSigned<decltype(d)> d_sel;
const auto sel_hi_mask = MaskFromVec(
BitCast(d_sel, VecFromMask(di32, DupEven(BitCast(di32, idx_vec)) >
Set(di32, int32_t{1}))));
const auto lo_lookup_result = BitCast(d_sel, TableLookupLanes(a, idx_mod));
const auto hi_lookup_result = BitCast(d_sel, TableLookupLanes(b, idx_mod));
return BitCast(d,
IfThenElse(sel_hi_mask, hi_lookup_result, lo_lookup_result));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> TwoTablesLookupLanes(Vec128<double> a, Vec128<double> b,
Indices128<double> idx) {
const DFromV<decltype(a)> d;
const RebindToSigned<decltype(d)> di;
const Vec128<int64_t> idx_vec{idx.raw};
const Indices128<double> idx_mod{And(idx_vec, Set(di, int64_t{1})).raw};
const Repartition<int32_t, decltype(d)> di32;
const auto sel_hi_mask =
MaskFromVec(BitCast(d, VecFromMask(di32, DupEven(BitCast(di32, idx_vec)) >
Set(di32, int32_t{1}))));
const auto lo_lookup_result = TableLookupLanes(a, idx_mod);
const auto hi_lookup_result = TableLookupLanes(b, idx_mod);
return IfThenElse(sel_hi_mask, hi_lookup_result, lo_lookup_result);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> OddEven(const Vec128<T, N> a, const Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const Repartition<uint8_t, decltype(d)> d8;
alignas(16) static constexpr uint8_t mask[16] = {
0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0};
return IfThenElse(MaskFromVec(BitCast(d, Load(d8, mask))), b, a);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> OddEven(const Vec128<T, N> a, const Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const Repartition<uint8_t, decltype(d)> d8;
alignas(16) static constexpr uint8_t mask[16] = {
0xFF, 0xFF, 0, 0, 0xFF, 0xFF, 0, 0, 0xFF, 0xFF, 0, 0, 0xFF, 0xFF, 0, 0};
return IfThenElse(MaskFromVec(BitCast(d, Load(d8, mask))), b, a);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> OddEven(const Vec128<T, N> a, const Vec128<T, N> b) {
const __m128i odd = _mm_shuffle_epi32(a.raw,
(((
3
) << 6) | ((
1
) << 4) | ((
3
) << 2) | (
1
))
);
const __m128i even = _mm_shuffle_epi32(b.raw,
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
);
return Vec128<T, N>{_mm_unpacklo_epi32(even, odd)};
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> OddEven(const Vec128<T, N> a, const Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const RebindToFloat<decltype(d)> dd;
return BitCast(
d, Vec128<double, N>{_mm_shuffle_pd(
BitCast(dd, b).raw, BitCast(dd, a).raw,
(((
1
) << 1) | (
0
))
)});
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> OddEven(Vec128<float, N> a, Vec128<float, N> b) {
const __m128 odd = _mm_shuffle_ps(a.raw, a.raw,
(((
3
) << 6) | ((
1
) << 4) | ((
3
) << 2) | (
1
))
);
const __m128 even = _mm_shuffle_ps(b.raw, b.raw,
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
);
return Vec128<float, N>{_mm_unpacklo_ps(even, odd)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> OddEvenBlocks(Vec128<T, N> , Vec128<T, N> even) {
return even;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> SwapAdjacentBlocks(Vec128<T, N> v) {
return v;
}
namespace detail {
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<MakeUnsigned<T>> Pow2(const Vec128<T> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const RepartitionToWide<decltype(d)> dw;
const Rebind<float, decltype(dw)> df;
const auto zero = Zero(d);
const auto exp = ShiftLeft<23 - 16>(v);
const auto upper = exp + Set(d, 0x3F80);
const auto f0 = ZipLower(dw, zero, upper);
const auto f1 = ZipUpper(dw, zero, upper);
const VFromD<decltype(dw)> bits0{_mm_cvtps_epi32(BitCast(df, f0).raw)};
const VFromD<decltype(dw)> bits1{_mm_cvtps_epi32(BitCast(df, f1).raw)};
return ConcatEven(du, BitCast(du, bits1), BitCast(du, bits0));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr, hwy::EnableIf<(N <= 4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<MakeUnsigned<T>, N> Pow2(const Vec128<T, N> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const Twice<decltype(du)> dt_u;
const RepartitionToWide<decltype(dt_u)> dt_w;
const RebindToFloat<decltype(dt_w)> dt_f;
const auto exp = ShiftLeft<23 - 16>(v);
const auto upper = exp + Set(d, 0x3F80);
const auto f0 = ZipLower(dt_w, Zero(dt_u), ResizeBitCast(dt_u, upper));
const VFromD<decltype(dt_w)> bits0{_mm_cvtps_epi32(BitCast(dt_f, f0).raw)};
alignas(16)
const uint16_t kCompactEvenU16[8] = {0x0100, 0x0504, 0x0908, 0x0D0C};
return TableLookupBytes(bits0, Load(du, kCompactEvenU16));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<MakeUnsigned<T>, N> Pow2(const Vec128<T, N> v) {
const DFromV<decltype(v)> d;
const auto exp = ShiftLeft<23>(v);
const auto f = exp + Set(d, 0x3F800000);
return Vec128<MakeUnsigned<T>, N>{_mm_cvtps_epi32(_mm_castsi128_ps(f.raw))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> Shl(hwy::UnsignedTag , Vec128<uint16_t, N> v,
Vec128<uint16_t, N> bits) {
return v * Pow2(bits);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec16<uint16_t> Shl(hwy::UnsignedTag , Vec16<uint16_t> v,
Vec16<uint16_t> bits) {
const auto bits16 = And(bits, Vec16<uint16_t>{_mm_set_epi64x(0, 0xFFFF)});
return Vec16<uint16_t>{_mm_sll_epi16(v.raw, bits16.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> Shl(hwy::UnsignedTag tag, Vec128<uint8_t, N> v,
Vec128<uint8_t, N> bits) {
const DFromV<decltype(v)> d;
const Repartition<uint16_t, decltype(d)> dw;
using VW = VFromD<decltype(dw)>;
const VW even_mask = Set(dw, 0x00FF);
const VW odd_mask = Set(dw, 0xFF00);
const VW vw = BitCast(dw, v);
const VW bits16 = BitCast(dw, bits);
const VW evens = Shl(tag, vw, And(bits16, even_mask));
const VW odds = Shl(tag, And(vw, odd_mask), ShiftRight<8>(bits16));
return OddEven(BitCast(d, odds), BitCast(d, evens));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, 1> Shl(hwy::UnsignedTag , Vec128<uint8_t, 1> v,
Vec128<uint8_t, 1> bits) {
const Vec16<uint16_t> bits8 =
And(Vec16<uint16_t>{bits.raw}, Vec16<uint16_t>{_mm_set_epi64x(0, 0xFF)});
return Vec128<uint8_t, 1>{_mm_sll_epi16(v.raw, bits8.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> Shl(hwy::UnsignedTag , Vec128<uint32_t, N> v,
Vec128<uint32_t, N> bits) {
return v * Pow2(bits);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<uint32_t> Shl(hwy::UnsignedTag , Vec32<uint32_t> v,
const Vec32<uint32_t> bits) {
const auto bits32 =
Combine(Full64<uint32_t>(), Zero(Full32<uint32_t>()), bits);
return Vec32<uint32_t>{_mm_sll_epi32(v.raw, bits32.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t> Shl(hwy::UnsignedTag , Vec128<uint64_t> v,
Vec128<uint64_t> bits) {
const DFromV<decltype(v)> d;
const Vec128<uint64_t> out0{_mm_sll_epi64(v.raw, bits.raw)};
const __m128i bits1 = _mm_unpackhi_epi64(bits.raw, bits.raw);
const Vec128<uint64_t> out1{_mm_sll_epi64(v.raw, bits1)};
return ConcatUpperLower(d, out1, out0);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<uint64_t> Shl(hwy::UnsignedTag , Vec64<uint64_t> v,
Vec64<uint64_t> bits) {
return Vec64<uint64_t>{_mm_sll_epi64(v.raw, bits.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Shl(hwy::SignedTag , Vec128<T, N> v,
Vec128<T, N> bits) {
const DFromV<decltype(v)> di;
const RebindToUnsigned<decltype(di)> du;
return BitCast(di,
Shl(hwy::UnsignedTag(), BitCast(du, v), BitCast(du, bits)));
}
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator<<(Vec128<T, N> v, Vec128<T, N> bits) {
return detail::Shl(hwy::TypeTag<T>(), v, bits);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> operator>>(Vec128<uint16_t, N> in,
const Vec128<uint16_t, N> bits) {
const DFromV<decltype(in)> d;
const auto out = MulHigh(in, detail::Pow2(Set(d, 16) - bits));
return IfThenElse(bits == Zero(d), in, out);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec16<uint16_t> operator>>(const Vec16<uint16_t> in,
const Vec16<uint16_t> bits) {
const auto bits16 = And(bits, Vec16<uint16_t>{_mm_set_epi64x(0, 0xFFFF)});
return Vec16<uint16_t>{_mm_srl_epi16(in.raw, bits16.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> operator>>(Vec128<uint8_t, N> in,
const Vec128<uint8_t, N> bits) {
const DFromV<decltype(in)> d;
const Repartition<uint16_t, decltype(d)> dw;
using VW = VFromD<decltype(dw)>;
const VW mask = Set(dw, 0x00FF);
const VW vw = BitCast(dw, in);
const VW bits16 = BitCast(dw, bits);
const VW evens = And(vw, mask) >> And(bits16, mask);
const VW odds = vw >> ShiftRight<8>(bits16);
return OddEven(BitCast(d, odds), BitCast(d, evens));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, 1> operator>>(const Vec128<uint8_t, 1> in,
const Vec128<uint8_t, 1> bits) {
const Vec16<uint16_t> mask{_mm_set_epi64x(0, 0xFF)};
const Vec16<uint16_t> in8 = And(Vec16<uint16_t>{in.raw}, mask);
const Vec16<uint16_t> bits8 = And(Vec16<uint16_t>{bits.raw}, mask);
return Vec128<uint8_t, 1>{_mm_srl_epi16(in8.raw, bits8.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> operator>>(const Vec128<uint32_t, N> in,
const Vec128<uint32_t, N> bits) {
const DFromV<decltype(in)> d32;
const Vec128<uint32_t, N> in31{_mm_shuffle_epi32(in.raw, 0x31)};
const auto mul = detail::Pow2(Set(d32, 32) - bits);
const auto out20 = ShiftRight<32>(MulEven(in, mul));
const Vec128<uint32_t, N> mul31{_mm_shuffle_epi32(mul.raw, 0x31)};
const auto out31 = BitCast(d32, MulEven(in31, mul31));
const Vec128<uint32_t, N> out = OddEven(out31, BitCast(d32, out20));
return IfThenElse(bits == Zero(d32), in, out);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, 1> operator>>(const Vec128<uint32_t, 1> in,
const Vec128<uint32_t, 1> bits) {
const auto bits32 =
Combine(Full64<uint32_t>(), Zero(Full32<uint32_t>()), bits);
return Vec128<uint32_t, 1>{_mm_srl_epi32(in.raw, bits32.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t> operator>>(const Vec128<uint64_t> v,
const Vec128<uint64_t> bits) {
const DFromV<decltype(v)> d;
const Vec128<uint64_t> out0{_mm_srl_epi64(v.raw, bits.raw)};
const __m128i bits1 = _mm_unpackhi_epi64(bits.raw, bits.raw);
const Vec128<uint64_t> out1{_mm_srl_epi64(v.raw, bits1)};
return ConcatUpperLower(d, out1, out0);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<uint64_t> operator>>(const Vec64<uint64_t> v,
const Vec64<uint64_t> bits) {
return Vec64<uint64_t>{_mm_srl_epi64(v.raw, bits.raw)};
}
namespace detail {
template <class DI, class V>
inline __attribute__((always_inline)) V SignedShr(const DI di, const V v, const V count_i) {
const RebindToUnsigned<DI> du;
const auto count = BitCast(du, count_i);
const auto sign = BroadcastSignBit(v);
const auto abs = BitCast(du, v ^ sign);
return BitCast(di, abs >> count) ^ sign;
}
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> operator>>(Vec128<int16_t, N> v,
Vec128<int16_t, N> bits) {
const DFromV<decltype(v)> d;
return detail::SignedShr(d, v, bits);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec16<int16_t> operator>>(Vec16<int16_t> v, Vec16<int16_t> bits) {
const auto bits16 = And(bits, Vec16<int16_t>{_mm_set_epi64x(0, 0xFFFF)});
return Vec16<int16_t>{_mm_sra_epi16(v.raw, bits16.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> operator>>(Vec128<int8_t, N> v,
Vec128<int8_t, N> bits) {
const DFromV<decltype(v)> d;
return detail::SignedShr(d, v, bits);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, 1> operator>>(Vec128<int8_t, 1> v,
Vec128<int8_t, 1> bits) {
const DFromV<decltype(v)> d;
const Rebind<int16_t, decltype(d)> di16;
const Twice<decltype(d)> dt;
const auto vi16 = ShiftRight<8>(BitCast(di16, Combine(dt, v, v)));
const Vec16<uint16_t> bits8 =
And(Vec16<uint16_t>{bits.raw}, Vec16<uint16_t>{_mm_set_epi64x(0, 0xFF)});
return Vec128<int8_t, 1>{_mm_sra_epi16(vi16.raw, bits8.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> operator>>(Vec128<int32_t, N> v,
Vec128<int32_t, N> bits) {
const DFromV<decltype(v)> d;
return detail::SignedShr(d, v, bits);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<int32_t> operator>>(Vec32<int32_t> v, Vec32<int32_t> bits) {
const auto bits32 = Combine(Full64<int32_t>(), Zero(Full32<int32_t>()), bits);
return Vec32<int32_t>{_mm_sra_epi32(v.raw, bits32.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> operator>>(Vec128<int64_t, N> v,
Vec128<int64_t, N> bits) {
const DFromV<decltype(v)> d;
return detail::SignedShr(d, v, bits);
}
inline __attribute__((always_inline)) Vec128<uint64_t> MulEven(Vec128<uint64_t> a, Vec128<uint64_t> b) {
const DFromV<decltype(a)> d;
alignas(16) uint64_t mul[2];
mul[0] = Mul128(GetLane(a), GetLane(b), &mul[1]);
return Load(d, mul);
}
inline __attribute__((always_inline)) Vec128<uint64_t> MulOdd(Vec128<uint64_t> a, Vec128<uint64_t> b) {
const DFromV<decltype(a)> d;
const Half<decltype(d)> d2;
alignas(16) uint64_t mul[2];
const uint64_t a1 = GetLane(UpperHalf(d2, a));
const uint64_t b1 = GetLane(UpperHalf(d2, b));
mul[0] = Mul128(a1, b1, &mul[1]);
return Load(d, mul);
}
template <class D32, hwy::EnableIf<IsSame<TFromD<D32>, float>()>* = nullptr,
class V16 = VFromD<Repartition<bfloat16_t, D32>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D32> WidenMulPairwiseAdd(D32 df32, V16 a, V16 b) {
const RebindToUnsigned<decltype(df32)> du32;
using VU32 = VFromD<decltype(du32)>;
const VU32 odd = Set(du32, 0xFFFF0000u);
const VU32 ae = ShiftLeft<16>(BitCast(du32, a));
const VU32 ao = And(BitCast(du32, a), odd);
const VU32 be = ShiftLeft<16>(BitCast(du32, b));
const VU32 bo = And(BitCast(du32, b), odd);
return MulAdd(BitCast(df32, ae), BitCast(df32, be),
Mul(BitCast(df32, ao), BitCast(df32, bo)));
}
template <class D32, hwy::EnableIf<IsSame<TFromD<D32>, int32_t>()>* = nullptr, hwy::EnableIf<D32::kPrivateLanes * sizeof(TFromD<D32>) <= 16>* = nullptr,
class V16 = VFromD<RepartitionToNarrow<D32>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D32> WidenMulPairwiseAdd(D32 , V16 a, V16 b) {
return VFromD<D32>{_mm_madd_epi16(a.raw, b.raw)};
}
template <class D32, hwy::EnableIf<IsSame<TFromD<D32>, float>()>* = nullptr,
class V16 = VFromD<Repartition<bfloat16_t, D32>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D32> ReorderWidenMulAccumulate(D32 df32, V16 a, V16 b,
const VFromD<D32> sum0,
VFromD<D32>& sum1) {
const RebindToUnsigned<decltype(df32)> du32;
using VU32 = VFromD<decltype(du32)>;
const VU32 odd = Set(du32, 0xFFFF0000u);
const VU32 ae = ShiftLeft<16>(BitCast(du32, a));
const VU32 ao = And(BitCast(du32, a), odd);
const VU32 be = ShiftLeft<16>(BitCast(du32, b));
const VU32 bo = And(BitCast(du32, b), odd);
sum1 = MulAdd(BitCast(df32, ao), BitCast(df32, bo), sum1);
return MulAdd(BitCast(df32, ae), BitCast(df32, be), sum0);
}
template <class D32, hwy::EnableIf<IsSame<TFromD<D32>, int32_t>()>* = nullptr, hwy::EnableIf<D32::kPrivateLanes * sizeof(TFromD<D32>) <= 16>* = nullptr,
class V16 = VFromD<RepartitionToNarrow<D32>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D32> ReorderWidenMulAccumulate(D32 d, V16 a, V16 b,
const VFromD<D32> sum0,
VFromD<D32>& ) {
(void)d;
return sum0 + WidenMulPairwiseAdd(d, a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> RearrangeToOddPlusEven(const Vec128<int32_t, N> sum0,
Vec128<int32_t, N> ) {
return sum0;
}
template <class VW>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VW RearrangeToOddPlusEven(const VW sum0, const VW sum1) {
return Add(sum0, sum1);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<uint8_t, D>> v) {
const __m128i zero = _mm_setzero_si128();
return VFromD<D>{_mm_unpacklo_epi8(v.raw, zero)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<uint16_t, D>> v) {
return VFromD<D>{_mm_unpacklo_epi16(v.raw, _mm_setzero_si128())};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<uint32_t, D>> v) {
return VFromD<D>{_mm_unpacklo_epi32(v.raw, _mm_setzero_si128())};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<uint8_t, D>> v) {
const __m128i zero = _mm_setzero_si128();
const __m128i u16 = _mm_unpacklo_epi8(v.raw, zero);
return VFromD<D>{_mm_unpacklo_epi16(u16, zero)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D d, VFromD<Rebind<uint8_t, D>> v) {
alignas(16) static constexpr int8_t kShuffle[16] = {
0, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1};
const Repartition<int8_t, decltype(d)> di8;
return TableLookupBytesOr0(v, BitCast(d, Load(di8, kShuffle)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D d, VFromD<Rebind<uint16_t, D>> v) {
alignas(16) static constexpr int8_t kShuffle[16] = {
0, 1, -1, -1, -1, -1, -1, -1, 2, 3, -1, -1, -1, -1, -1, -1};
const Repartition<int8_t, decltype(d)> di8;
return TableLookupBytesOr0(v, BitCast(d, Load(di8, kShuffle)));
}
template <class D, class V, hwy::EnableIf<IsSigned<TFromD<D> >() && !IsFloat<TFromD<D> >() && !IsSpecialFloat<TFromD<D> >()>* = nullptr, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
hwy::EnableIf<(sizeof(TFromD<D>) > sizeof(TFromV<V>))>* = nullptr,
hwy::EnableIf<(D::kPrivateLanes == DFromV<V>::kPrivateLanes)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D di, V v) {
const RebindToUnsigned<decltype(di)> du;
return BitCast(di, PromoteTo(du, v));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<int8_t, D>> v) {
return ShiftRight<8>(VFromD<D>{_mm_unpacklo_epi8(v.raw, v.raw)});
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<int16_t, D>> v) {
return ShiftRight<16>(VFromD<D>{_mm_unpacklo_epi16(v.raw, v.raw)});
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<int32_t, D>> v) {
return ShiftRight<32>(VFromD<D>{_mm_unpacklo_epi32(v.raw, v.raw)});
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<int8_t, D>> v) {
const __m128i x2 = _mm_unpacklo_epi8(v.raw, v.raw);
const __m128i x4 = _mm_unpacklo_epi16(x2, x2);
return ShiftRight<24>(VFromD<D>{x4});
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D d, VFromD<Rebind<int8_t, D>> v) {
const Repartition<int32_t, decltype(d)> di32;
const Half<decltype(di32)> dh_i32;
const VFromD<decltype(di32)> x4{PromoteTo(dh_i32, v).raw};
const VFromD<decltype(di32)> s4{
_mm_shufflelo_epi16(x4.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
return ZipLower(d, x4, s4);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D d, VFromD<Rebind<int16_t, D>> v) {
const Repartition<int32_t, decltype(d)> di32;
const Half<decltype(di32)> dh_i32;
const VFromD<decltype(di32)> x2{PromoteTo(dh_i32, v).raw};
const VFromD<decltype(di32)> s2{
_mm_shufflelo_epi16(x2.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
return ZipLower(d, x2, s2);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> PromoteTo(D df32, VFromD<Rebind<float16_t, D>> v) {
const RebindToSigned<decltype(df32)> di32;
const RebindToUnsigned<decltype(df32)> du32;
const auto bits16 = PromoteTo(du32, VFromD<Rebind<uint16_t, D>>{v.raw});
const auto sign = ShiftRight<15>(bits16);
const auto biased_exp = ShiftRight<10>(bits16) & Set(du32, 0x1F);
const auto mantissa = bits16 & Set(du32, 0x3FF);
const auto subnormal =
BitCast(du32, ConvertTo(df32, BitCast(di32, mantissa)) *
Set(df32, 1.0f / 16384 / 1024));
const auto biased_exp32 = biased_exp + Set(du32, 127 - 15);
const auto mantissa32 = ShiftLeft<23 - 10>(mantissa);
const auto normal = ShiftLeft<23>(biased_exp32) | mantissa32;
const auto bits32 = IfThenElse(biased_exp == Zero(du32), subnormal, normal);
return BitCast(df32, ShiftLeft<31>(sign) | bits32);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D df32, VFromD<Rebind<bfloat16_t, D>> v) {
const Rebind<uint16_t, decltype(df32)> du16;
const RebindToSigned<decltype(df32)> di32;
return BitCast(df32, ShiftLeft<16>(PromoteTo(di32, BitCast(du16, v))));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<float, D>> v) {
return VFromD<D>{_mm_cvtps_pd(v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<int32_t, D>> v) {
return VFromD<D>{_mm_cvtepi32_pd(v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int32_t, D>> v) {
return VFromD<D>{_mm_packs_epi32(v.raw, v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int32_t, D>> v) {
const Rebind<int32_t, D> di32;
const auto zero_if_neg = AndNot(ShiftRight<31>(v), v);
const auto too_big = VecFromMask(di32, Gt(v, Set(di32, 0xFFFF)));
const auto clamped = Or(zero_if_neg, too_big);
const Repartition<uint16_t, decltype(di32)> du16;
alignas(16) static constexpr uint16_t kLower2Bytes[16] = {
0x0100, 0x0504, 0x0908, 0x0D0C, 0x8080, 0x8080, 0x8080, 0x8080};
const auto lo2 = Load(du16, kLower2Bytes);
return VFromD<D>{TableLookupBytes(BitCast(du16, clamped), lo2).raw};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D du16, VFromD<Rebind<uint32_t, D>> v) {
const DFromV<decltype(v)> du32;
const RebindToSigned<decltype(du32)> di32;
const auto too_big =
VecFromMask(di32, Gt(BitCast(di32, ShiftRight<16>(v)), Zero(di32)));
const auto clamped = Or(BitCast(di32, v), too_big);
(void)du16;
const Repartition<uint16_t, decltype(di32)> du16_full;
alignas(16) static constexpr uint16_t kLower2Bytes[16] = {
0x0100, 0x0504, 0x0908, 0x0D0C, 0x8080, 0x8080, 0x8080, 0x8080};
const auto lo2 = Load(du16_full, kLower2Bytes);
return VFromD<D>{TableLookupBytes(BitCast(du16_full, clamped), lo2).raw};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int32_t, D>> v) {
const __m128i i16 = _mm_packs_epi32(v.raw, v.raw);
return VFromD<D>{_mm_packus_epi16(i16, i16)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int16_t, D>> v) {
return VFromD<D>{_mm_packus_epi16(v.raw, v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int32_t, D>> v) {
const __m128i i16 = _mm_packs_epi32(v.raw, v.raw);
return VFromD<D>{_mm_packs_epi16(i16, i16)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int16_t, D>> v) {
return VFromD<D>{_mm_packs_epi16(v.raw, v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D du8, VFromD<Rebind<uint32_t, D>> v) {
const DFromV<decltype(v)> du32;
const RebindToSigned<decltype(du32)> di32;
const auto max_i32 = Set(du32, 0x7FFFFFFFu);
const Repartition<uint8_t, decltype(du32)> du32_as_du8;
const auto clamped = BitCast(
di32, Min(BitCast(du32_as_du8, v), BitCast(du32_as_du8, max_i32)));
return DemoteTo(du8, clamped);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D du8, VFromD<Rebind<uint16_t, D>> v) {
const DFromV<decltype(v)> du16;
const RebindToSigned<decltype(du16)> di16;
const auto max_i16 = Set(du16, 0x7FFF);
const Repartition<uint8_t, decltype(du16)> du16_as_du8;
const auto clamped = BitCast(
di16, Min(BitCast(du16_as_du8, v), BitCast(du16_as_du8, max_i16)));
return DemoteTo(du8, clamped);
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmain"
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D df16, VFromD<Rebind<float, D>> v) {
const RebindToUnsigned<decltype(df16)> du16;
const Rebind<uint32_t, decltype(df16)> du;
const RebindToSigned<decltype(du)> di;
const auto bits32 = BitCast(du, v);
const auto sign = ShiftRight<31>(bits32);
const auto biased_exp32 = ShiftRight<23>(bits32) & Set(du, 0xFF);
const auto mantissa32 = bits32 & Set(du, 0x7FFFFF);
const auto k15 = Set(di, 15);
const auto exp = Min(BitCast(di, biased_exp32) - Set(di, 127), k15);
const auto is_tiny = exp < Set(di, -24);
const auto is_subnormal = exp < Set(di, -14);
const auto biased_exp16 =
BitCast(du, IfThenZeroElse(is_subnormal, exp + k15));
const auto sub_exp = BitCast(du, Set(di, -14) - exp);
const auto sub_m = (Set(du, 1) << (Set(du, 10) - sub_exp)) +
(mantissa32 >> (Set(du, 13) + sub_exp));
const auto mantissa16 = IfThenElse(RebindMask(du, is_subnormal), sub_m,
ShiftRight<13>(mantissa32));
const auto sign16 = ShiftLeft<15>(sign);
const auto normal16 = sign16 | ShiftLeft<10>(biased_exp16) | mantissa16;
const auto bits16 = IfThenZeroElse(is_tiny, BitCast(di, normal16));
return BitCast(df16, DemoteTo(du16, bits16));
}
#pragma GCC diagnostic pop
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, bfloat16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D dbf16, VFromD<Rebind<float, D>> v) {
const Rebind<int32_t, decltype(dbf16)> di32;
const Rebind<uint32_t, decltype(dbf16)> du32;
const Rebind<uint16_t, decltype(dbf16)> du16;
const auto bits_in_32 = BitCast(di32, ShiftRight<16>(BitCast(du32, v)));
return BitCast(dbf16, DemoteTo(du16, bits_in_32));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, bfloat16_t>()>* = nullptr,
class V32 = VFromD<Repartition<float, D>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dbf16, V32 a, V32 b) {
const RebindToUnsigned<decltype(dbf16)> du16;
const Repartition<uint32_t, decltype(dbf16)> du32;
const VFromD<decltype(du32)> b_in_even = ShiftRight<16>(BitCast(du32, b));
return BitCast(dbf16, OddEven(BitCast(du16, a), BitCast(du16, b_in_even)));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<int16_t> ReorderDemote2To(D dn, Vec32<int32_t> a,
Vec32<int32_t> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<int16_t> ReorderDemote2To(D , Vec64<int32_t> a,
Vec64<int32_t> b) {
return Vec64<int16_t>{_mm_shuffle_epi32(_mm_packs_epi32(a.raw, b.raw),
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t> ReorderDemote2To(D , Vec128<int32_t> a,
Vec128<int32_t> b) {
return Vec128<int16_t>{_mm_packs_epi32(a.raw, b.raw)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<uint16_t> ReorderDemote2To(D dn, Vec32<int32_t> a,
Vec32<int32_t> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<uint16_t> ReorderDemote2To(D dn, Vec64<int32_t> a,
Vec64<int32_t> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t> ReorderDemote2To(D dn, Vec128<int32_t> a,
Vec128<int32_t> b) {
const Half<decltype(dn)> dnh;
const auto u16_a = DemoteTo(dnh, a);
const auto u16_b = DemoteTo(dnh, b);
return Combine(dn, u16_b, u16_a);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, Vec128<uint32_t> a,
Vec128<uint32_t> b) {
const DFromV<decltype(a)> du32;
const RebindToSigned<decltype(du32)> di32;
const auto max_i32 = Set(du32, 0x7FFFFFFFu);
const Repartition<uint8_t, decltype(du32)> du32_as_du8;
const auto clamped_a = BitCast(
di32, Min(BitCast(du32_as_du8, a), BitCast(du32_as_du8, max_i32)));
const auto clamped_b = BitCast(
di32, Min(BitCast(du32_as_du8, b), BitCast(du32_as_du8, max_i32)));
return ReorderDemote2To(dn, clamped_a, clamped_b);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<uint32_t, D>> a,
VFromD<Repartition<uint32_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<int16_t, D>> a,
VFromD<Repartition<int16_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<int8_t> ReorderDemote2To(D , Vec64<int16_t> a,
Vec64<int16_t> b) {
return Vec64<int8_t>{_mm_shuffle_epi32(_mm_packs_epi16(a.raw, b.raw),
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t> ReorderDemote2To(D , Vec128<int16_t> a,
Vec128<int16_t> b) {
return Vec128<int8_t>{_mm_packs_epi16(a.raw, b.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<int16_t, D>> a,
VFromD<Repartition<int16_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<uint8_t> ReorderDemote2To(D , Vec64<int16_t> a,
Vec64<int16_t> b) {
return Vec64<uint8_t>{_mm_shuffle_epi32(_mm_packus_epi16(a.raw, b.raw),
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t> ReorderDemote2To(D , Vec128<int16_t> a,
Vec128<int16_t> b) {
return Vec128<uint8_t>{_mm_packus_epi16(a.raw, b.raw)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, Vec128<uint16_t> a,
Vec128<uint16_t> b) {
const DFromV<decltype(a)> du16;
const RebindToSigned<decltype(du16)> di16;
const auto max_i16 = Set(du16, 0x7FFFu);
const Repartition<uint8_t, decltype(du16)> du16_as_du8;
const auto clamped_a = BitCast(
di16, Min(BitCast(du16_as_du8, a), BitCast(du16_as_du8, max_i16)));
const auto clamped_b = BitCast(
di16, Min(BitCast(du16_as_du8, b), BitCast(du16_as_du8, max_i16)));
return ReorderDemote2To(dn, clamped_a, clamped_b);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<uint16_t, D>> a,
VFromD<Repartition<uint16_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<!hwy::IsFloat<TFromD<D> >() && !hwy::IsSpecialFloat<TFromD<D> >()>* = nullptr,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr,
hwy::EnableIf<sizeof(TFromV<V>) == (sizeof(TFromD<D>) * 2)>* = nullptr,
hwy::EnableIf<(D::kPrivateLanes == DFromV<V>::kPrivateLanes * 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> OrderedDemote2To(D d, V a, V b) {
return ReorderDemote2To(d, a, b);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, bfloat16_t>()>* = nullptr, class V32 = VFromD<Repartition<float, D>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> OrderedDemote2To(D dbf16, V32 a, V32 b) {
const RebindToUnsigned<decltype(dbf16)> du16;
return BitCast(dbf16, ConcatOdd(du16, BitCast(du16, b), BitCast(du16, a)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<double, D>> v) {
return VFromD<D>{_mm_cvtpd_ps(v.raw)};
}
namespace detail {
template <class D>
inline __attribute__((always_inline)) VFromD<D> ClampF64ToI32Max(D d, VFromD<D> v) {
return Min(v, Set(d, 2147483647.0));
}
template <class DI, class DF = RebindToFloat<DI>>
inline __attribute__((always_inline)) VFromD<DI> FixConversionOverflow(
DI di, VFromD<DF> original, decltype(Zero(DI()).raw) converted_raw) {
const VFromD<DI> converted{converted_raw};
const VFromD<DI> sign_wrong = AndNot(BitCast(di, original), converted);
const RebindToUnsigned<DI> du;
const VFromD<DI> mask = BroadcastSignBit(sign_wrong);
const VFromD<DI> max = BitCast(di, ShiftRight<1>(BitCast(du, mask)));
return IfVecThenElse(mask, max, converted);
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr,
class DF = Rebind<double, D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<DF> v) {
const VFromD<DF> clamped = detail::ClampF64ToI32Max(DF(), v);
return VFromD<D>{_mm_cvttpd_epi32(clamped.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> U8FromU32(const Vec128<uint32_t, N> v) {
const DFromV<decltype(v)> d32;
const Repartition<uint8_t, decltype(d32)> d8;
alignas(16) static constexpr uint32_t k8From32[4] = {
0x0C080400u, 0x0C080400u, 0x0C080400u, 0x0C080400u};
const auto quad = TableLookupBytes(v, Load(d32, k8From32));
return LowerHalf(LowerHalf(BitCast(d8, quad)));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> MulFixedPoint15(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_mulhrs_epi16(a.raw, b.raw)};
}
template <typename From, class DTo, hwy::EnableIf<(DTo::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DTo> TruncateTo(DTo , Vec128<From, 1> v) {
const Repartition<TFromD<DTo>, DFromV<decltype(v)>> dto;
return VFromD<DTo>{BitCast(dto, v).raw};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec16<uint8_t> TruncateTo(D d, Vec128<uint64_t> v) {
const Repartition<uint8_t, DFromV<decltype(v)>> d8;
(void)d;
alignas(16) static constexpr uint8_t kIdx[16] = {0, 8, 0, 8, 0, 8, 0, 8,
0, 8, 0, 8, 0, 8, 0, 8};
const Vec128<uint8_t> v8 = TableLookupBytes(v, Load(d8, kIdx));
return LowerHalf(LowerHalf(LowerHalf(v8)));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<uint16_t> TruncateTo(D d, Vec128<uint64_t> v) {
(void)d;
const Repartition<uint16_t, DFromV<decltype(v)>> d16;
alignas(16) static constexpr uint16_t kIdx[8] = {
0x100u, 0x908u, 0x100u, 0x908u, 0x100u, 0x908u, 0x100u, 0x908u};
const Vec128<uint16_t> v16 = TableLookupBytes(v, Load(d16, kIdx));
return LowerHalf(LowerHalf(v16));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<uint32_t> TruncateTo(D , Vec128<uint64_t> v) {
return Vec64<uint32_t>{_mm_shuffle_epi32(v.raw, 0x88)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TruncateTo(D , VFromD<Rebind<uint32_t, D>> v) {
const DFromV<decltype(v)> du32;
const Repartition<uint8_t, decltype(du32)> d;
alignas(16) static constexpr uint8_t kIdx[16] = {
0x0u, 0x4u, 0x8u, 0xCu, 0x0u, 0x4u, 0x8u, 0xCu,
0x0u, 0x4u, 0x8u, 0xCu, 0x0u, 0x4u, 0x8u, 0xCu};
return LowerHalf(LowerHalf(TableLookupBytes(v, Load(d, kIdx))));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TruncateTo(D , VFromD<Rebind<uint32_t, D>> v) {
const DFromV<decltype(v)> du32;
const Repartition<uint16_t, decltype(du32)> d;
return LowerHalf(ConcatEven(d, BitCast(d, v), BitCast(d, v)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TruncateTo(D , VFromD<Rebind<uint16_t, D>> v) {
const DFromV<decltype(v)> du16;
const Repartition<uint8_t, decltype(du16)> d;
return LowerHalf(ConcatEven(d, BitCast(d, v), BitCast(d, v)));
}
namespace detail {
template <class D, hwy::EnableIf<!IsSigned<TFromD<D> >()>* = nullptr>
inline __attribute__((always_inline)) VFromD<Rebind<uint64_t, D>> DemoteFromU64MaskOutResult(
D , VFromD<Rebind<uint64_t, D>> v) {
return v;
}
template <class D, hwy::EnableIf<IsSigned<TFromD<D> >() && !IsFloat<TFromD<D> >() && !IsSpecialFloat<TFromD<D> >()>* = nullptr>
inline __attribute__((always_inline)) VFromD<Rebind<uint64_t, D>> DemoteFromU64MaskOutResult(
D , VFromD<Rebind<uint64_t, D>> v) {
const DFromV<decltype(v)> du64;
return And(v,
Set(du64, static_cast<uint64_t>(hwy::HighestValue<TFromD<D>>())));
}
template <class D>
inline __attribute__((always_inline)) VFromD<Rebind<uint64_t, D>> DemoteFromU64Saturate(
D dn, VFromD<Rebind<uint64_t, D>> v) {
const Rebind<uint64_t, D> du64;
const RebindToSigned<decltype(du64)> di64;
constexpr int kShiftAmt = static_cast<int>(sizeof(TFromD<D>) * 8) -
static_cast<int>(hwy::IsSigned<TFromD<D>>());
const auto too_big = BitCast(
du64, VecFromMask(
di64, Gt(BitCast(di64, ShiftRight<kShiftAmt>(v)), Zero(di64))));
return DemoteFromU64MaskOutResult(dn, Or(v, too_big));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, class V>
inline __attribute__((always_inline)) VFromD<D> ReorderDemote2From64To32Combine(D dn, V a, V b) {
return ConcatEven(dn, BitCast(dn, b), BitCast(dn, a));
}
}
template <class D, hwy::EnableIf<((size_t{1} << sizeof(TFromD<D>)) & ((1 << 1) | (1 << 2) | (1 << 4))) != 0>* = nullptr,
hwy::EnableIf<IsSigned<TFromD<D> >() && !IsFloat<TFromD<D> >() && !IsSpecialFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D dn, VFromD<Rebind<int64_t, D>> v) {
const DFromV<decltype(v)> di64;
const RebindToUnsigned<decltype(di64)> du64;
const RebindToUnsigned<decltype(dn)> dn_u;
const auto invert_mask = BitCast(du64, BroadcastSignBit(v));
const auto saturated_vals = Xor(
invert_mask,
detail::DemoteFromU64Saturate(dn, Xor(invert_mask, BitCast(du64, v))));
return BitCast(dn, TruncateTo(dn_u, saturated_vals));
}
template <class D, hwy::EnableIf<((size_t{1} << sizeof(TFromD<D>)) & ((1 << 1) | (1 << 2) | (1 << 4))) != 0>* = nullptr,
hwy::EnableIf<!IsSigned<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D dn, VFromD<Rebind<int64_t, D>> v) {
const DFromV<decltype(v)> di64;
const RebindToUnsigned<decltype(di64)> du64;
const auto non_neg_vals = BitCast(du64, AndNot(BroadcastSignBit(v), v));
return TruncateTo(dn, detail::DemoteFromU64Saturate(dn, non_neg_vals));
}
template <class D, hwy::EnableIf<((size_t{1} << sizeof(TFromD<D>)) & ((1 << 1) | (1 << 2) | (1 << 4))) != 0>* = nullptr,
hwy::EnableIf<!IsSigned<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D dn, VFromD<Rebind<uint64_t, D>> v) {
return TruncateTo(dn, detail::DemoteFromU64Saturate(dn, v));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16 / 2>* = nullptr,
hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr, hwy::EnableIf<!hwy::IsFloat<TFromD<D> >() && !hwy::IsSpecialFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<int64_t, D>> a,
VFromD<Repartition<int64_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16 / 2>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<uint64_t, D>> a,
VFromD<Repartition<uint64_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> ReorderDemote2To(D dn, Vec128<int64_t> a,
Vec128<int64_t> b) {
const DFromV<decltype(a)> di64;
const RebindToUnsigned<decltype(di64)> du64;
const Half<decltype(dn)> dnh;
const auto invert_mask_a = BitCast(du64, BroadcastSignBit(a));
const auto invert_mask_b = BitCast(du64, BroadcastSignBit(b));
const auto saturated_a = Xor(
invert_mask_a,
detail::DemoteFromU64Saturate(dnh, Xor(invert_mask_a, BitCast(du64, a))));
const auto saturated_b = Xor(
invert_mask_b,
detail::DemoteFromU64Saturate(dnh, Xor(invert_mask_b, BitCast(du64, b))));
return ConcatEven(dn, BitCast(dn, saturated_b), BitCast(dn, saturated_a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> ReorderDemote2To(D dn, Vec128<int64_t> a,
Vec128<int64_t> b) {
const DFromV<decltype(a)> di64;
const RebindToUnsigned<decltype(di64)> du64;
const Half<decltype(dn)> dnh;
const auto saturated_a = detail::DemoteFromU64Saturate(
dnh, BitCast(du64, AndNot(BroadcastSignBit(a), a)));
const auto saturated_b = detail::DemoteFromU64Saturate(
dnh, BitCast(du64, AndNot(BroadcastSignBit(b), b)));
return ConcatEven(dn, BitCast(dn, saturated_b), BitCast(dn, saturated_a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> ReorderDemote2To(D dn, Vec128<uint64_t> a,
Vec128<uint64_t> b) {
const Half<decltype(dn)> dnh;
const auto saturated_a = detail::DemoteFromU64Saturate(dnh, a);
const auto saturated_b = detail::DemoteFromU64Saturate(dnh, b);
return ConcatEven(dn, BitCast(dn, saturated_b), BitCast(dn, saturated_a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConvertTo(D , VFromD<Rebind<int32_t, D>> v) {
return VFromD<D>{_mm_cvtepi32_ps(v.raw)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConvertTo(D df, VFromD<Rebind<uint32_t, D>> v) {
const RebindToUnsigned<decltype(df)> du32;
const RebindToSigned<decltype(df)> d32;
const auto msk_lo = Set(du32, 0xFFFF);
const auto cnst2_16_flt = Set(df, 65536.0f);
const auto v_lo = BitCast(d32, And(v, msk_lo));
const auto v_hi = BitCast(d32, ShiftRight<16>(v));
return MulAdd(cnst2_16_flt, ConvertTo(df, v_hi), ConvertTo(df, v_lo));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConvertTo(D dd, VFromD<Rebind<int64_t, D>> v) {
const Repartition<uint32_t, decltype(dd)> d32;
const Repartition<uint64_t, decltype(dd)> d64;
const auto k84_63 = Set(d64, 0x4530000080000000ULL);
const auto v_upper = BitCast(dd, ShiftRight<32>(BitCast(d64, v)) ^ k84_63);
const auto k52 = Set(d32, 0x43300000);
const auto v_lower = BitCast(dd, OddEven(k52, BitCast(d32, v)));
const auto k84_63_52 = BitCast(dd, Set(d64, 0x4530000080100000ULL));
return (v_upper - k84_63_52) + v_lower;
}
namespace detail {
template <class VW>
inline __attribute__((always_inline)) VFromD<Rebind<double, DFromV<VW>>> U64ToF64VecFast(VW w) {
const DFromV<decltype(w)> d64;
const RebindToFloat<decltype(d64)> dd;
const auto cnst2_52_dbl = Set(dd, 0x0010000000000000);
return BitCast(dd, Or(w, BitCast(d64, cnst2_52_dbl))) - cnst2_52_dbl;
}
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConvertTo(D dd, VFromD<Rebind<uint64_t, D>> v) {
const RebindToUnsigned<decltype(dd)> d64;
using VU = VFromD<decltype(d64)>;
const VU msk_lo = Set(d64, 0xFFFFFFFF);
const auto cnst2_32_dbl = Set(dd, 4294967296.0);
const VU v_lo = And(v, msk_lo);
const VU v_hi = ShiftRight<32>(v);
const auto v_lo_dbl = detail::U64ToF64VecFast(v_lo);
return MulAdd(cnst2_32_dbl, detail::U64ToF64VecFast(v_hi), v_lo_dbl);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConvertTo(D di, VFromD<Rebind<float, D>> v) {
return detail::FixConversionOverflow(di, v, _mm_cvttps_epi32(v.raw));
}
template <class DI, hwy::EnableIf<(DI::kPrivateLanes * sizeof(TFromD<DI>) > (0 ? 16 : 0))>* = nullptr,
hwy::EnableIf<IsSame<TFromD<DI>, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DI> ConvertTo(DI di, VFromD<Rebind<double, DI>> v) {
using VI = VFromD<decltype(di)>;
const RebindToUnsigned<decltype(di)> du;
using VU = VFromD<decltype(du)>;
const Repartition<uint16_t, decltype(di)> du16;
const VI k1075 = Set(di, 1075);
const VU biased_exp = ShiftRight<52>(BitCast(du, v)) & Set(du, 0x7FF);
const Repartition<int32_t, decltype(di)> di32;
const auto in_range = MaskFromVec(BitCast(
di,
VecFromMask(di32, DupEven(BitCast(di32, biased_exp)) < Set(di32, 1086))));
const VU shift_mnt = BitCast(
du, SaturatedSub(BitCast(du16, k1075), BitCast(du16, biased_exp)));
const VU shift_int = BitCast(
du, SaturatedSub(BitCast(du16, biased_exp), BitCast(du16, k1075)));
const VU mantissa = BitCast(du, v) & Set(du, (1ULL << 52) - 1);
const VU int53 = (mantissa | Set(du, 1ULL << 52)) >> shift_mnt;
const VU shifted = int53 << shift_int;
const VI sign_mask = BroadcastSignBit(BitCast(di, v));
const VI limit = Set(di, LimitsMax<int64_t>()) - sign_mask;
const VI magnitude = IfThenElse(in_range, BitCast(di, shifted), limit);
return (magnitude ^ sign_mask) - sign_mask;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> NearestInt(const Vec128<float, N> v) {
const RebindToSigned<DFromV<decltype(v)>> di;
return detail::FixConversionOverflow(di, v, _mm_cvtps_epi32(v.raw));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Round(const Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> df;
const auto max = Set(df, MantissaEnd<T>());
const auto large = CopySignToAbs(max, v);
const auto added = large + v;
const auto rounded = added - large;
return IfThenElse(Abs(v) < max, rounded, v);
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) Mask128<T, N> UseInt(const Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> d;
return Abs(v) < Set(d, MantissaEnd<T>());
}
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Trunc(const Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> df;
const RebindToSigned<decltype(df)> di;
const auto integer = ConvertTo(di, v);
const auto int_f = ConvertTo(df, integer);
return IfThenElse(detail::UseInt(v), CopySign(int_f, v), v);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Ceil(const Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> df;
const RebindToSigned<decltype(df)> di;
const auto integer = ConvertTo(di, v);
const auto int_f = ConvertTo(df, integer);
const auto neg1 = ConvertTo(df, VecFromMask(di, RebindMask(di, int_f < v)));
return IfThenElse(detail::UseInt(v), int_f - neg1, v);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Floor(const Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> df;
const RebindToSigned<decltype(df)> di;
const auto integer = ConvertTo(di, v);
const auto int_f = ConvertTo(df, integer);
const auto neg1 = ConvertTo(df, VecFromMask(di, RebindMask(di, int_f > v)));
return IfThenElse(detail::UseInt(v), int_f + neg1, v);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<float, N> IsNaN(const Vec128<float, N> v) {
return Mask128<float, N>{_mm_cmpunord_ps(v.raw, v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<double, N> IsNaN(const Vec128<double, N> v) {
return Mask128<double, N>{_mm_cmpunord_pd(v.raw, v.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> IsInf(const Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> d;
const RebindToSigned<decltype(d)> di;
const VFromD<decltype(di)> vi = BitCast(di, v);
return RebindMask(d, Eq(Add(vi, vi), Set(di, hwy::MaxExponentTimes2<T>())));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> IsFinite(const Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const RebindToSigned<decltype(d)> di;
const VFromD<decltype(du)> vu = BitCast(du, v);
const VFromD<decltype(di)> exp =
BitCast(di, ShiftRight<hwy::MantissaBits<T>() + 1>(ShiftLeft<1>(vu)));
return RebindMask(d, Lt(exp, Set(di, hwy::MaxExponentField<T>())));
}
namespace detail {
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
inline __attribute__((always_inline)) MFromD<D> LoadMaskBits128(D d, uint64_t mask_bits) {
const RebindToUnsigned<decltype(d)> du;
const VFromD<D> vbits{_mm_cvtsi32_si128(static_cast<int>(mask_bits))};
alignas(16) static constexpr uint8_t kRep8[16] = {0, 0, 0, 0, 0, 0, 0, 0,
1, 1, 1, 1, 1, 1, 1, 1};
const auto rep8 = TableLookupBytes(vbits, Load(du, kRep8));
alignas(16) static constexpr uint8_t kBit[16] = {1, 2, 4, 8, 16, 32, 64, 128,
1, 2, 4, 8, 16, 32, 64, 128};
return RebindMask(d, TestBit(rep8, LoadDup128(du, kBit)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
inline __attribute__((always_inline)) MFromD<D> LoadMaskBits128(D d, uint64_t mask_bits) {
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint16_t kBit[8] = {1, 2, 4, 8, 16, 32, 64, 128};
const auto vmask_bits = Set(du, static_cast<uint16_t>(mask_bits));
return RebindMask(d, TestBit(vmask_bits, Load(du, kBit)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
inline __attribute__((always_inline)) MFromD<D> LoadMaskBits128(D d, uint64_t mask_bits) {
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint32_t kBit[8] = {1, 2, 4, 8};
const auto vmask_bits = Set(du, static_cast<uint32_t>(mask_bits));
return RebindMask(d, TestBit(vmask_bits, Load(du, kBit)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
inline __attribute__((always_inline)) MFromD<D> LoadMaskBits128(D d, uint64_t mask_bits) {
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint64_t kBit[8] = {1, 2};
return RebindMask(d, TestBit(Set(du, mask_bits), Load(du, kBit)));
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> LoadMaskBits(D d, const uint8_t* __restrict__ bits) {
constexpr size_t kN = MaxLanes(d);
uint64_t mask_bits = 0;
constexpr size_t kNumBytes = (kN + 7) / 8;
CopyBytes<kNumBytes>(bits, &mask_bits);
if (kN < 8) {
mask_bits &= (1ull << kN) - 1;
}
return detail::LoadMaskBits128(d, mask_bits);
}
template <typename T>
struct CompressIsPartition {
enum { value = (sizeof(T) != 1) };
};
namespace detail {
constexpr inline __attribute__((always_inline)) uint64_t U64FromInt(int mask_bits) {
return static_cast<uint64_t>(static_cast<unsigned>(mask_bits));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) uint64_t BitsFromMask(hwy::SizeTag<1> ,
const Mask128<T, N> mask) {
const Simd<T, N, 0> d;
const auto sign_bits = BitCast(d, VecFromMask(d, mask)).raw;
return U64FromInt(_mm_movemask_epi8(sign_bits));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) uint64_t BitsFromMask(hwy::SizeTag<2> ,
const Mask128<T, N> mask) {
const auto sign_bits = _mm_packs_epi16(mask.raw, _mm_setzero_si128());
return U64FromInt(_mm_movemask_epi8(sign_bits));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) uint64_t BitsFromMask(hwy::SizeTag<4> , Mask128<T, N> mask) {
const Simd<T, N, 0> d;
const Simd<float, N, 0> df;
const auto sign_bits = BitCast(df, VecFromMask(d, mask));
return U64FromInt(_mm_movemask_ps(sign_bits.raw));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) uint64_t BitsFromMask(hwy::SizeTag<8> , Mask128<T, N> mask) {
const Simd<T, N, 0> d;
const Simd<double, N, 0> df;
const auto sign_bits = BitCast(df, VecFromMask(d, mask));
return U64FromInt(_mm_movemask_pd(sign_bits.raw));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) uint64_t BitsFromMask(const Mask128<T, N> mask) {
return OnlyActive<T, N>(BitsFromMask(hwy::SizeTag<sizeof(T)>(), mask));
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t StoreMaskBits(D d, MFromD<D> mask, uint8_t* bits) {
constexpr size_t kNumBytes = (MaxLanes(d) + 7) / 8;
const uint64_t mask_bits = detail::BitsFromMask(mask);
CopyBytes<kNumBytes>(&mask_bits, bits);
return kNumBytes;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) bool AllFalse(D , MFromD<D> mask) {
return detail::BitsFromMask(mask) == 0;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) bool AllTrue(D d, MFromD<D> mask) {
constexpr uint64_t kAllBits = (1ull << MaxLanes(d)) - 1;
return detail::BitsFromMask(mask) == kAllBits;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CountTrue(D , MFromD<D> mask) {
return PopCount(detail::BitsFromMask(mask));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t FindKnownFirstTrue(D , MFromD<D> mask) {
return Num0BitsBelowLS1Bit_Nonzero32(
static_cast<uint32_t>(detail::BitsFromMask(mask)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) intptr_t FindFirstTrue(D , MFromD<D> mask) {
const uint32_t mask_bits = static_cast<uint32_t>(detail::BitsFromMask(mask));
return mask_bits ? intptr_t(Num0BitsBelowLS1Bit_Nonzero32(mask_bits)) : -1;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t FindKnownLastTrue(D , MFromD<D> mask) {
return 31 - Num0BitsAboveMS1Bit_Nonzero32(
static_cast<uint32_t>(detail::BitsFromMask(mask)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) intptr_t FindLastTrue(D , MFromD<D> mask) {
const uint32_t mask_bits = static_cast<uint32_t>(detail::BitsFromMask(mask));
return mask_bits ? intptr_t(31 - Num0BitsAboveMS1Bit_Nonzero32(mask_bits))
: -1;
}
namespace detail {
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromBits128(D d, uint64_t mask_bits) {
do { } while (0);
const Rebind<uint8_t, decltype(d)> d8;
const Twice<decltype(d8)> d8t;
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint8_t table[2048] = {
0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
2, 0, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
4, 0, 2, 6, 8, 10, 12, 14, 0, 4, 2, 6, 8, 10, 12, 14,
2, 4, 0, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
6, 0, 2, 4, 8, 10, 12, 14, 0, 6, 2, 4, 8, 10, 12, 14,
2, 6, 0, 4, 8, 10, 12, 14, 0, 2, 6, 4, 8, 10, 12, 14,
4, 6, 0, 2, 8, 10, 12, 14, 0, 4, 6, 2, 8, 10, 12, 14,
2, 4, 6, 0, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
8, 0, 2, 4, 6, 10, 12, 14, 0, 8, 2, 4, 6, 10, 12, 14,
2, 8, 0, 4, 6, 10, 12, 14, 0, 2, 8, 4, 6, 10, 12, 14,
4, 8, 0, 2, 6, 10, 12, 14, 0, 4, 8, 2, 6, 10, 12, 14,
2, 4, 8, 0, 6, 10, 12, 14, 0, 2, 4, 8, 6, 10, 12, 14,
6, 8, 0, 2, 4, 10, 12, 14, 0, 6, 8, 2, 4, 10, 12, 14,
2, 6, 8, 0, 4, 10, 12, 14, 0, 2, 6, 8, 4, 10, 12, 14,
4, 6, 8, 0, 2, 10, 12, 14, 0, 4, 6, 8, 2, 10, 12, 14,
2, 4, 6, 8, 0, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
10, 0, 2, 4, 6, 8, 12, 14, 0, 10, 2, 4, 6, 8, 12, 14,
2, 10, 0, 4, 6, 8, 12, 14, 0, 2, 10, 4, 6, 8, 12, 14,
4, 10, 0, 2, 6, 8, 12, 14, 0, 4, 10, 2, 6, 8, 12, 14,
2, 4, 10, 0, 6, 8, 12, 14, 0, 2, 4, 10, 6, 8, 12, 14,
6, 10, 0, 2, 4, 8, 12, 14, 0, 6, 10, 2, 4, 8, 12, 14,
2, 6, 10, 0, 4, 8, 12, 14, 0, 2, 6, 10, 4, 8, 12, 14,
4, 6, 10, 0, 2, 8, 12, 14, 0, 4, 6, 10, 2, 8, 12, 14,
2, 4, 6, 10, 0, 8, 12, 14, 0, 2, 4, 6, 10, 8, 12, 14,
8, 10, 0, 2, 4, 6, 12, 14, 0, 8, 10, 2, 4, 6, 12, 14,
2, 8, 10, 0, 4, 6, 12, 14, 0, 2, 8, 10, 4, 6, 12, 14,
4, 8, 10, 0, 2, 6, 12, 14, 0, 4, 8, 10, 2, 6, 12, 14,
2, 4, 8, 10, 0, 6, 12, 14, 0, 2, 4, 8, 10, 6, 12, 14,
6, 8, 10, 0, 2, 4, 12, 14, 0, 6, 8, 10, 2, 4, 12, 14,
2, 6, 8, 10, 0, 4, 12, 14, 0, 2, 6, 8, 10, 4, 12, 14,
4, 6, 8, 10, 0, 2, 12, 14, 0, 4, 6, 8, 10, 2, 12, 14,
2, 4, 6, 8, 10, 0, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
12, 0, 2, 4, 6, 8, 10, 14, 0, 12, 2, 4, 6, 8, 10, 14,
2, 12, 0, 4, 6, 8, 10, 14, 0, 2, 12, 4, 6, 8, 10, 14,
4, 12, 0, 2, 6, 8, 10, 14, 0, 4, 12, 2, 6, 8, 10, 14,
2, 4, 12, 0, 6, 8, 10, 14, 0, 2, 4, 12, 6, 8, 10, 14,
6, 12, 0, 2, 4, 8, 10, 14, 0, 6, 12, 2, 4, 8, 10, 14,
2, 6, 12, 0, 4, 8, 10, 14, 0, 2, 6, 12, 4, 8, 10, 14,
4, 6, 12, 0, 2, 8, 10, 14, 0, 4, 6, 12, 2, 8, 10, 14,
2, 4, 6, 12, 0, 8, 10, 14, 0, 2, 4, 6, 12, 8, 10, 14,
8, 12, 0, 2, 4, 6, 10, 14, 0, 8, 12, 2, 4, 6, 10, 14,
2, 8, 12, 0, 4, 6, 10, 14, 0, 2, 8, 12, 4, 6, 10, 14,
4, 8, 12, 0, 2, 6, 10, 14, 0, 4, 8, 12, 2, 6, 10, 14,
2, 4, 8, 12, 0, 6, 10, 14, 0, 2, 4, 8, 12, 6, 10, 14,
6, 8, 12, 0, 2, 4, 10, 14, 0, 6, 8, 12, 2, 4, 10, 14,
2, 6, 8, 12, 0, 4, 10, 14, 0, 2, 6, 8, 12, 4, 10, 14,
4, 6, 8, 12, 0, 2, 10, 14, 0, 4, 6, 8, 12, 2, 10, 14,
2, 4, 6, 8, 12, 0, 10, 14, 0, 2, 4, 6, 8, 12, 10, 14,
10, 12, 0, 2, 4, 6, 8, 14, 0, 10, 12, 2, 4, 6, 8, 14,
2, 10, 12, 0, 4, 6, 8, 14, 0, 2, 10, 12, 4, 6, 8, 14,
4, 10, 12, 0, 2, 6, 8, 14, 0, 4, 10, 12, 2, 6, 8, 14,
2, 4, 10, 12, 0, 6, 8, 14, 0, 2, 4, 10, 12, 6, 8, 14,
6, 10, 12, 0, 2, 4, 8, 14, 0, 6, 10, 12, 2, 4, 8, 14,
2, 6, 10, 12, 0, 4, 8, 14, 0, 2, 6, 10, 12, 4, 8, 14,
4, 6, 10, 12, 0, 2, 8, 14, 0, 4, 6, 10, 12, 2, 8, 14,
2, 4, 6, 10, 12, 0, 8, 14, 0, 2, 4, 6, 10, 12, 8, 14,
8, 10, 12, 0, 2, 4, 6, 14, 0, 8, 10, 12, 2, 4, 6, 14,
2, 8, 10, 12, 0, 4, 6, 14, 0, 2, 8, 10, 12, 4, 6, 14,
4, 8, 10, 12, 0, 2, 6, 14, 0, 4, 8, 10, 12, 2, 6, 14,
2, 4, 8, 10, 12, 0, 6, 14, 0, 2, 4, 8, 10, 12, 6, 14,
6, 8, 10, 12, 0, 2, 4, 14, 0, 6, 8, 10, 12, 2, 4, 14,
2, 6, 8, 10, 12, 0, 4, 14, 0, 2, 6, 8, 10, 12, 4, 14,
4, 6, 8, 10, 12, 0, 2, 14, 0, 4, 6, 8, 10, 12, 2, 14,
2, 4, 6, 8, 10, 12, 0, 14, 0, 2, 4, 6, 8, 10, 12, 14,
14, 0, 2, 4, 6, 8, 10, 12, 0, 14, 2, 4, 6, 8, 10, 12,
2, 14, 0, 4, 6, 8, 10, 12, 0, 2, 14, 4, 6, 8, 10, 12,
4, 14, 0, 2, 6, 8, 10, 12, 0, 4, 14, 2, 6, 8, 10, 12,
2, 4, 14, 0, 6, 8, 10, 12, 0, 2, 4, 14, 6, 8, 10, 12,
6, 14, 0, 2, 4, 8, 10, 12, 0, 6, 14, 2, 4, 8, 10, 12,
2, 6, 14, 0, 4, 8, 10, 12, 0, 2, 6, 14, 4, 8, 10, 12,
4, 6, 14, 0, 2, 8, 10, 12, 0, 4, 6, 14, 2, 8, 10, 12,
2, 4, 6, 14, 0, 8, 10, 12, 0, 2, 4, 6, 14, 8, 10, 12,
8, 14, 0, 2, 4, 6, 10, 12, 0, 8, 14, 2, 4, 6, 10, 12,
2, 8, 14, 0, 4, 6, 10, 12, 0, 2, 8, 14, 4, 6, 10, 12,
4, 8, 14, 0, 2, 6, 10, 12, 0, 4, 8, 14, 2, 6, 10, 12,
2, 4, 8, 14, 0, 6, 10, 12, 0, 2, 4, 8, 14, 6, 10, 12,
6, 8, 14, 0, 2, 4, 10, 12, 0, 6, 8, 14, 2, 4, 10, 12,
2, 6, 8, 14, 0, 4, 10, 12, 0, 2, 6, 8, 14, 4, 10, 12,
4, 6, 8, 14, 0, 2, 10, 12, 0, 4, 6, 8, 14, 2, 10, 12,
2, 4, 6, 8, 14, 0, 10, 12, 0, 2, 4, 6, 8, 14, 10, 12,
10, 14, 0, 2, 4, 6, 8, 12, 0, 10, 14, 2, 4, 6, 8, 12,
2, 10, 14, 0, 4, 6, 8, 12, 0, 2, 10, 14, 4, 6, 8, 12,
4, 10, 14, 0, 2, 6, 8, 12, 0, 4, 10, 14, 2, 6, 8, 12,
2, 4, 10, 14, 0, 6, 8, 12, 0, 2, 4, 10, 14, 6, 8, 12,
6, 10, 14, 0, 2, 4, 8, 12, 0, 6, 10, 14, 2, 4, 8, 12,
2, 6, 10, 14, 0, 4, 8, 12, 0, 2, 6, 10, 14, 4, 8, 12,
4, 6, 10, 14, 0, 2, 8, 12, 0, 4, 6, 10, 14, 2, 8, 12,
2, 4, 6, 10, 14, 0, 8, 12, 0, 2, 4, 6, 10, 14, 8, 12,
8, 10, 14, 0, 2, 4, 6, 12, 0, 8, 10, 14, 2, 4, 6, 12,
2, 8, 10, 14, 0, 4, 6, 12, 0, 2, 8, 10, 14, 4, 6, 12,
4, 8, 10, 14, 0, 2, 6, 12, 0, 4, 8, 10, 14, 2, 6, 12,
2, 4, 8, 10, 14, 0, 6, 12, 0, 2, 4, 8, 10, 14, 6, 12,
6, 8, 10, 14, 0, 2, 4, 12, 0, 6, 8, 10, 14, 2, 4, 12,
2, 6, 8, 10, 14, 0, 4, 12, 0, 2, 6, 8, 10, 14, 4, 12,
4, 6, 8, 10, 14, 0, 2, 12, 0, 4, 6, 8, 10, 14, 2, 12,
2, 4, 6, 8, 10, 14, 0, 12, 0, 2, 4, 6, 8, 10, 14, 12,
12, 14, 0, 2, 4, 6, 8, 10, 0, 12, 14, 2, 4, 6, 8, 10,
2, 12, 14, 0, 4, 6, 8, 10, 0, 2, 12, 14, 4, 6, 8, 10,
4, 12, 14, 0, 2, 6, 8, 10, 0, 4, 12, 14, 2, 6, 8, 10,
2, 4, 12, 14, 0, 6, 8, 10, 0, 2, 4, 12, 14, 6, 8, 10,
6, 12, 14, 0, 2, 4, 8, 10, 0, 6, 12, 14, 2, 4, 8, 10,
2, 6, 12, 14, 0, 4, 8, 10, 0, 2, 6, 12, 14, 4, 8, 10,
4, 6, 12, 14, 0, 2, 8, 10, 0, 4, 6, 12, 14, 2, 8, 10,
2, 4, 6, 12, 14, 0, 8, 10, 0, 2, 4, 6, 12, 14, 8, 10,
8, 12, 14, 0, 2, 4, 6, 10, 0, 8, 12, 14, 2, 4, 6, 10,
2, 8, 12, 14, 0, 4, 6, 10, 0, 2, 8, 12, 14, 4, 6, 10,
4, 8, 12, 14, 0, 2, 6, 10, 0, 4, 8, 12, 14, 2, 6, 10,
2, 4, 8, 12, 14, 0, 6, 10, 0, 2, 4, 8, 12, 14, 6, 10,
6, 8, 12, 14, 0, 2, 4, 10, 0, 6, 8, 12, 14, 2, 4, 10,
2, 6, 8, 12, 14, 0, 4, 10, 0, 2, 6, 8, 12, 14, 4, 10,
4, 6, 8, 12, 14, 0, 2, 10, 0, 4, 6, 8, 12, 14, 2, 10,
2, 4, 6, 8, 12, 14, 0, 10, 0, 2, 4, 6, 8, 12, 14, 10,
10, 12, 14, 0, 2, 4, 6, 8, 0, 10, 12, 14, 2, 4, 6, 8,
2, 10, 12, 14, 0, 4, 6, 8, 0, 2, 10, 12, 14, 4, 6, 8,
4, 10, 12, 14, 0, 2, 6, 8, 0, 4, 10, 12, 14, 2, 6, 8,
2, 4, 10, 12, 14, 0, 6, 8, 0, 2, 4, 10, 12, 14, 6, 8,
6, 10, 12, 14, 0, 2, 4, 8, 0, 6, 10, 12, 14, 2, 4, 8,
2, 6, 10, 12, 14, 0, 4, 8, 0, 2, 6, 10, 12, 14, 4, 8,
4, 6, 10, 12, 14, 0, 2, 8, 0, 4, 6, 10, 12, 14, 2, 8,
2, 4, 6, 10, 12, 14, 0, 8, 0, 2, 4, 6, 10, 12, 14, 8,
8, 10, 12, 14, 0, 2, 4, 6, 0, 8, 10, 12, 14, 2, 4, 6,
2, 8, 10, 12, 14, 0, 4, 6, 0, 2, 8, 10, 12, 14, 4, 6,
4, 8, 10, 12, 14, 0, 2, 6, 0, 4, 8, 10, 12, 14, 2, 6,
2, 4, 8, 10, 12, 14, 0, 6, 0, 2, 4, 8, 10, 12, 14, 6,
6, 8, 10, 12, 14, 0, 2, 4, 0, 6, 8, 10, 12, 14, 2, 4,
2, 6, 8, 10, 12, 14, 0, 4, 0, 2, 6, 8, 10, 12, 14, 4,
4, 6, 8, 10, 12, 14, 0, 2, 0, 4, 6, 8, 10, 12, 14, 2,
2, 4, 6, 8, 10, 12, 14, 0, 0, 2, 4, 6, 8, 10, 12, 14};
const VFromD<decltype(d8t)> byte_idx{Load(d8, table + mask_bits * 8).raw};
const VFromD<decltype(du)> pairs = ZipLower(byte_idx, byte_idx);
return BitCast(d, pairs + Set(du, 0x0100));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromNotBits128(D d, uint64_t mask_bits) {
do { } while (0);
const Rebind<uint8_t, decltype(d)> d8;
const Twice<decltype(d8)> d8t;
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint8_t table[2048] = {
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0,
0, 4, 6, 8, 10, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2,
0, 2, 6, 8, 10, 12, 14, 4, 2, 6, 8, 10, 12, 14, 0, 4,
0, 6, 8, 10, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4,
0, 2, 4, 8, 10, 12, 14, 6, 2, 4, 8, 10, 12, 14, 0, 6,
0, 4, 8, 10, 12, 14, 2, 6, 4, 8, 10, 12, 14, 0, 2, 6,
0, 2, 8, 10, 12, 14, 4, 6, 2, 8, 10, 12, 14, 0, 4, 6,
0, 8, 10, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6,
0, 2, 4, 6, 10, 12, 14, 8, 2, 4, 6, 10, 12, 14, 0, 8,
0, 4, 6, 10, 12, 14, 2, 8, 4, 6, 10, 12, 14, 0, 2, 8,
0, 2, 6, 10, 12, 14, 4, 8, 2, 6, 10, 12, 14, 0, 4, 8,
0, 6, 10, 12, 14, 2, 4, 8, 6, 10, 12, 14, 0, 2, 4, 8,
0, 2, 4, 10, 12, 14, 6, 8, 2, 4, 10, 12, 14, 0, 6, 8,
0, 4, 10, 12, 14, 2, 6, 8, 4, 10, 12, 14, 0, 2, 6, 8,
0, 2, 10, 12, 14, 4, 6, 8, 2, 10, 12, 14, 0, 4, 6, 8,
0, 10, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8,
0, 2, 4, 6, 8, 12, 14, 10, 2, 4, 6, 8, 12, 14, 0, 10,
0, 4, 6, 8, 12, 14, 2, 10, 4, 6, 8, 12, 14, 0, 2, 10,
0, 2, 6, 8, 12, 14, 4, 10, 2, 6, 8, 12, 14, 0, 4, 10,
0, 6, 8, 12, 14, 2, 4, 10, 6, 8, 12, 14, 0, 2, 4, 10,
0, 2, 4, 8, 12, 14, 6, 10, 2, 4, 8, 12, 14, 0, 6, 10,
0, 4, 8, 12, 14, 2, 6, 10, 4, 8, 12, 14, 0, 2, 6, 10,
0, 2, 8, 12, 14, 4, 6, 10, 2, 8, 12, 14, 0, 4, 6, 10,
0, 8, 12, 14, 2, 4, 6, 10, 8, 12, 14, 0, 2, 4, 6, 10,
0, 2, 4, 6, 12, 14, 8, 10, 2, 4, 6, 12, 14, 0, 8, 10,
0, 4, 6, 12, 14, 2, 8, 10, 4, 6, 12, 14, 0, 2, 8, 10,
0, 2, 6, 12, 14, 4, 8, 10, 2, 6, 12, 14, 0, 4, 8, 10,
0, 6, 12, 14, 2, 4, 8, 10, 6, 12, 14, 0, 2, 4, 8, 10,
0, 2, 4, 12, 14, 6, 8, 10, 2, 4, 12, 14, 0, 6, 8, 10,
0, 4, 12, 14, 2, 6, 8, 10, 4, 12, 14, 0, 2, 6, 8, 10,
0, 2, 12, 14, 4, 6, 8, 10, 2, 12, 14, 0, 4, 6, 8, 10,
0, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10,
0, 2, 4, 6, 8, 10, 14, 12, 2, 4, 6, 8, 10, 14, 0, 12,
0, 4, 6, 8, 10, 14, 2, 12, 4, 6, 8, 10, 14, 0, 2, 12,
0, 2, 6, 8, 10, 14, 4, 12, 2, 6, 8, 10, 14, 0, 4, 12,
0, 6, 8, 10, 14, 2, 4, 12, 6, 8, 10, 14, 0, 2, 4, 12,
0, 2, 4, 8, 10, 14, 6, 12, 2, 4, 8, 10, 14, 0, 6, 12,
0, 4, 8, 10, 14, 2, 6, 12, 4, 8, 10, 14, 0, 2, 6, 12,
0, 2, 8, 10, 14, 4, 6, 12, 2, 8, 10, 14, 0, 4, 6, 12,
0, 8, 10, 14, 2, 4, 6, 12, 8, 10, 14, 0, 2, 4, 6, 12,
0, 2, 4, 6, 10, 14, 8, 12, 2, 4, 6, 10, 14, 0, 8, 12,
0, 4, 6, 10, 14, 2, 8, 12, 4, 6, 10, 14, 0, 2, 8, 12,
0, 2, 6, 10, 14, 4, 8, 12, 2, 6, 10, 14, 0, 4, 8, 12,
0, 6, 10, 14, 2, 4, 8, 12, 6, 10, 14, 0, 2, 4, 8, 12,
0, 2, 4, 10, 14, 6, 8, 12, 2, 4, 10, 14, 0, 6, 8, 12,
0, 4, 10, 14, 2, 6, 8, 12, 4, 10, 14, 0, 2, 6, 8, 12,
0, 2, 10, 14, 4, 6, 8, 12, 2, 10, 14, 0, 4, 6, 8, 12,
0, 10, 14, 2, 4, 6, 8, 12, 10, 14, 0, 2, 4, 6, 8, 12,
0, 2, 4, 6, 8, 14, 10, 12, 2, 4, 6, 8, 14, 0, 10, 12,
0, 4, 6, 8, 14, 2, 10, 12, 4, 6, 8, 14, 0, 2, 10, 12,
0, 2, 6, 8, 14, 4, 10, 12, 2, 6, 8, 14, 0, 4, 10, 12,
0, 6, 8, 14, 2, 4, 10, 12, 6, 8, 14, 0, 2, 4, 10, 12,
0, 2, 4, 8, 14, 6, 10, 12, 2, 4, 8, 14, 0, 6, 10, 12,
0, 4, 8, 14, 2, 6, 10, 12, 4, 8, 14, 0, 2, 6, 10, 12,
0, 2, 8, 14, 4, 6, 10, 12, 2, 8, 14, 0, 4, 6, 10, 12,
0, 8, 14, 2, 4, 6, 10, 12, 8, 14, 0, 2, 4, 6, 10, 12,
0, 2, 4, 6, 14, 8, 10, 12, 2, 4, 6, 14, 0, 8, 10, 12,
0, 4, 6, 14, 2, 8, 10, 12, 4, 6, 14, 0, 2, 8, 10, 12,
0, 2, 6, 14, 4, 8, 10, 12, 2, 6, 14, 0, 4, 8, 10, 12,
0, 6, 14, 2, 4, 8, 10, 12, 6, 14, 0, 2, 4, 8, 10, 12,
0, 2, 4, 14, 6, 8, 10, 12, 2, 4, 14, 0, 6, 8, 10, 12,
0, 4, 14, 2, 6, 8, 10, 12, 4, 14, 0, 2, 6, 8, 10, 12,
0, 2, 14, 4, 6, 8, 10, 12, 2, 14, 0, 4, 6, 8, 10, 12,
0, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12,
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 6, 8, 10, 12, 0, 14,
0, 4, 6, 8, 10, 12, 2, 14, 4, 6, 8, 10, 12, 0, 2, 14,
0, 2, 6, 8, 10, 12, 4, 14, 2, 6, 8, 10, 12, 0, 4, 14,
0, 6, 8, 10, 12, 2, 4, 14, 6, 8, 10, 12, 0, 2, 4, 14,
0, 2, 4, 8, 10, 12, 6, 14, 2, 4, 8, 10, 12, 0, 6, 14,
0, 4, 8, 10, 12, 2, 6, 14, 4, 8, 10, 12, 0, 2, 6, 14,
0, 2, 8, 10, 12, 4, 6, 14, 2, 8, 10, 12, 0, 4, 6, 14,
0, 8, 10, 12, 2, 4, 6, 14, 8, 10, 12, 0, 2, 4, 6, 14,
0, 2, 4, 6, 10, 12, 8, 14, 2, 4, 6, 10, 12, 0, 8, 14,
0, 4, 6, 10, 12, 2, 8, 14, 4, 6, 10, 12, 0, 2, 8, 14,
0, 2, 6, 10, 12, 4, 8, 14, 2, 6, 10, 12, 0, 4, 8, 14,
0, 6, 10, 12, 2, 4, 8, 14, 6, 10, 12, 0, 2, 4, 8, 14,
0, 2, 4, 10, 12, 6, 8, 14, 2, 4, 10, 12, 0, 6, 8, 14,
0, 4, 10, 12, 2, 6, 8, 14, 4, 10, 12, 0, 2, 6, 8, 14,
0, 2, 10, 12, 4, 6, 8, 14, 2, 10, 12, 0, 4, 6, 8, 14,
0, 10, 12, 2, 4, 6, 8, 14, 10, 12, 0, 2, 4, 6, 8, 14,
0, 2, 4, 6, 8, 12, 10, 14, 2, 4, 6, 8, 12, 0, 10, 14,
0, 4, 6, 8, 12, 2, 10, 14, 4, 6, 8, 12, 0, 2, 10, 14,
0, 2, 6, 8, 12, 4, 10, 14, 2, 6, 8, 12, 0, 4, 10, 14,
0, 6, 8, 12, 2, 4, 10, 14, 6, 8, 12, 0, 2, 4, 10, 14,
0, 2, 4, 8, 12, 6, 10, 14, 2, 4, 8, 12, 0, 6, 10, 14,
0, 4, 8, 12, 2, 6, 10, 14, 4, 8, 12, 0, 2, 6, 10, 14,
0, 2, 8, 12, 4, 6, 10, 14, 2, 8, 12, 0, 4, 6, 10, 14,
0, 8, 12, 2, 4, 6, 10, 14, 8, 12, 0, 2, 4, 6, 10, 14,
0, 2, 4, 6, 12, 8, 10, 14, 2, 4, 6, 12, 0, 8, 10, 14,
0, 4, 6, 12, 2, 8, 10, 14, 4, 6, 12, 0, 2, 8, 10, 14,
0, 2, 6, 12, 4, 8, 10, 14, 2, 6, 12, 0, 4, 8, 10, 14,
0, 6, 12, 2, 4, 8, 10, 14, 6, 12, 0, 2, 4, 8, 10, 14,
0, 2, 4, 12, 6, 8, 10, 14, 2, 4, 12, 0, 6, 8, 10, 14,
0, 4, 12, 2, 6, 8, 10, 14, 4, 12, 0, 2, 6, 8, 10, 14,
0, 2, 12, 4, 6, 8, 10, 14, 2, 12, 0, 4, 6, 8, 10, 14,
0, 12, 2, 4, 6, 8, 10, 14, 12, 0, 2, 4, 6, 8, 10, 14,
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 6, 8, 10, 0, 12, 14,
0, 4, 6, 8, 10, 2, 12, 14, 4, 6, 8, 10, 0, 2, 12, 14,
0, 2, 6, 8, 10, 4, 12, 14, 2, 6, 8, 10, 0, 4, 12, 14,
0, 6, 8, 10, 2, 4, 12, 14, 6, 8, 10, 0, 2, 4, 12, 14,
0, 2, 4, 8, 10, 6, 12, 14, 2, 4, 8, 10, 0, 6, 12, 14,
0, 4, 8, 10, 2, 6, 12, 14, 4, 8, 10, 0, 2, 6, 12, 14,
0, 2, 8, 10, 4, 6, 12, 14, 2, 8, 10, 0, 4, 6, 12, 14,
0, 8, 10, 2, 4, 6, 12, 14, 8, 10, 0, 2, 4, 6, 12, 14,
0, 2, 4, 6, 10, 8, 12, 14, 2, 4, 6, 10, 0, 8, 12, 14,
0, 4, 6, 10, 2, 8, 12, 14, 4, 6, 10, 0, 2, 8, 12, 14,
0, 2, 6, 10, 4, 8, 12, 14, 2, 6, 10, 0, 4, 8, 12, 14,
0, 6, 10, 2, 4, 8, 12, 14, 6, 10, 0, 2, 4, 8, 12, 14,
0, 2, 4, 10, 6, 8, 12, 14, 2, 4, 10, 0, 6, 8, 12, 14,
0, 4, 10, 2, 6, 8, 12, 14, 4, 10, 0, 2, 6, 8, 12, 14,
0, 2, 10, 4, 6, 8, 12, 14, 2, 10, 0, 4, 6, 8, 12, 14,
0, 10, 2, 4, 6, 8, 12, 14, 10, 0, 2, 4, 6, 8, 12, 14,
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 6, 8, 0, 10, 12, 14,
0, 4, 6, 8, 2, 10, 12, 14, 4, 6, 8, 0, 2, 10, 12, 14,
0, 2, 6, 8, 4, 10, 12, 14, 2, 6, 8, 0, 4, 10, 12, 14,
0, 6, 8, 2, 4, 10, 12, 14, 6, 8, 0, 2, 4, 10, 12, 14,
0, 2, 4, 8, 6, 10, 12, 14, 2, 4, 8, 0, 6, 10, 12, 14,
0, 4, 8, 2, 6, 10, 12, 14, 4, 8, 0, 2, 6, 10, 12, 14,
0, 2, 8, 4, 6, 10, 12, 14, 2, 8, 0, 4, 6, 10, 12, 14,
0, 8, 2, 4, 6, 10, 12, 14, 8, 0, 2, 4, 6, 10, 12, 14,
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 6, 0, 8, 10, 12, 14,
0, 4, 6, 2, 8, 10, 12, 14, 4, 6, 0, 2, 8, 10, 12, 14,
0, 2, 6, 4, 8, 10, 12, 14, 2, 6, 0, 4, 8, 10, 12, 14,
0, 6, 2, 4, 8, 10, 12, 14, 6, 0, 2, 4, 8, 10, 12, 14,
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 0, 6, 8, 10, 12, 14,
0, 4, 2, 6, 8, 10, 12, 14, 4, 0, 2, 6, 8, 10, 12, 14,
0, 2, 4, 6, 8, 10, 12, 14, 2, 0, 4, 6, 8, 10, 12, 14,
0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14};
const VFromD<decltype(d8t)> byte_idx{Load(d8, table + mask_bits * 8).raw};
const VFromD<decltype(du)> pairs = ZipLower(byte_idx, byte_idx);
return BitCast(d, pairs + Set(du, 0x0100));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromBits128(D d, uint64_t mask_bits) {
do { } while (0);
alignas(16) static constexpr uint8_t u8_indices[256] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
8, 9, 10, 11, 0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15,
0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15,
4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 12, 13, 14, 15,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7, 8, 9, 10, 11,
4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 8, 9, 10, 11,
0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 8, 9, 10, 11,
8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15, 4, 5, 6, 7,
4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, Load(d8, u8_indices + 16 * mask_bits));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromNotBits128(D d, uint64_t mask_bits) {
do { } while (0);
alignas(16) static constexpr uint8_t u8_indices[256] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 5,
6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3,
8, 9, 10, 11, 12, 13, 14, 15, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15, 0, 1,
2, 3, 8, 9, 10, 11, 0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7,
8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
10, 11, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 12, 13, 14, 15, 0, 1,
2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15, 8, 9, 10, 11,
0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5,
6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 5, 6, 7, 0, 1, 2, 3,
8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
12, 13, 14, 15};
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, Load(d8, u8_indices + 16 * mask_bits));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromBits128(D d, uint64_t mask_bits) {
do { } while (0);
alignas(16) static constexpr uint8_t u8_indices[64] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, Load(d8, u8_indices + 16 * mask_bits));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromNotBits128(D d, uint64_t mask_bits) {
do { } while (0);
alignas(16) static constexpr uint8_t u8_indices[64] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, Load(d8, u8_indices + 16 * mask_bits));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> CompressBits(Vec128<T, N> v, uint64_t mask_bits) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
do { } while (0);
const auto indices = BitCast(du, detail::IndicesFromBits128(d, mask_bits));
return BitCast(d, TableLookupBytes(BitCast(du, v), indices));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> CompressNotBits(Vec128<T, N> v, uint64_t mask_bits) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
do { } while (0);
const auto indices = BitCast(du, detail::IndicesFromNotBits128(d, mask_bits));
return BitCast(d, TableLookupBytes(BitCast(du, v), indices));
}
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> Compress(Vec128<T, 1> v, Mask128<T, 1> ) {
return v;
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Compress(Vec128<T> v, Mask128<T> mask) {
const DFromV<decltype(v)> d;
const Vec128<T> m = VecFromMask(d, mask);
const Vec128<T> maskL = DupEven(m);
const Vec128<T> maskH = DupOdd(m);
const Vec128<T> swap = AndNot(maskL, maskH);
return IfVecThenElse(swap, Shuffle01(v), v);
}
template <typename T, size_t N, hwy::EnableIf<((size_t{1} << sizeof(T)) & ((1 << 2) | (1 << 4))) != 0>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Compress(Vec128<T, N> v, Mask128<T, N> mask) {
return detail::CompressBits(v, detail::BitsFromMask(mask));
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> CompressNot(Vec128<T, 1> v, Mask128<T, 1> ) {
return v;
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> CompressNot(Vec128<T> v, Mask128<T> mask) {
const DFromV<decltype(v)> d;
const Vec128<T> m = VecFromMask(d, mask);
const Vec128<T> maskL = DupEven(m);
const Vec128<T> maskH = DupOdd(m);
const Vec128<T> swap = AndNot(maskH, maskL);
return IfVecThenElse(swap, Shuffle01(v), v);
}
template <typename T, size_t N, hwy::EnableIf<((size_t{1} << sizeof(T)) & ((1 << 2) | (1 << 4))) != 0>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> CompressNot(Vec128<T, N> v, Mask128<T, N> mask) {
if (N < 16 / sizeof(T)) {
return detail::CompressBits(v, detail::BitsFromMask(Not(mask)));
}
return detail::CompressNotBits(v, detail::BitsFromMask(mask));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t> CompressBlocksNot(Vec128<uint64_t> v,
Mask128<uint64_t> ) {
return v;
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> CompressBits(Vec128<T, N> v,
const uint8_t* __restrict__ bits) {
uint64_t mask_bits = 0;
constexpr size_t kNumBytes = (N + 7) / 8;
CopyBytes<kNumBytes>(bits, &mask_bits);
if (N < 8) {
mask_bits &= (1ull << N) - 1;
}
return detail::CompressBits(v, mask_bits);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressStore(VFromD<D> v, MFromD<D> m, D d,
TFromD<D>* __restrict__ unaligned) {
const RebindToUnsigned<decltype(d)> du;
const uint64_t mask_bits = detail::BitsFromMask(m);
do { } while (0);
const size_t count = PopCount(mask_bits);
const auto indices = BitCast(du, detail::IndicesFromBits128(d, mask_bits));
const auto compressed = BitCast(d, TableLookupBytes(BitCast(du, v), indices));
StoreU(compressed, d, unaligned);
detail::MaybeUnpoison(unaligned, count);
return count;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressBlendedStore(VFromD<D> v, MFromD<D> m, D d,
TFromD<D>* __restrict__ unaligned) {
const RebindToUnsigned<decltype(d)> du;
const uint64_t mask_bits = detail::BitsFromMask(m);
do { } while (0);
const size_t count = PopCount(mask_bits);
const auto indices = BitCast(du, detail::IndicesFromBits128(d, mask_bits));
const auto compressed = BitCast(d, TableLookupBytes(BitCast(du, v), indices));
BlendedStore(compressed, FirstN(d, count), d, unaligned);
detail::MaybeUnpoison(unaligned, count);
return count;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressBitsStore(VFromD<D> v, const uint8_t* __restrict__ bits,
D d, TFromD<D>* __restrict__ unaligned) {
const RebindToUnsigned<decltype(d)> du;
uint64_t mask_bits = 0;
constexpr size_t kN = MaxLanes(d);
constexpr size_t kNumBytes = (kN + 7) / 8;
CopyBytes<kNumBytes>(bits, &mask_bits);
if (kN < 8) {
mask_bits &= (1ull << kN) - 1;
}
const size_t count = PopCount(mask_bits);
const auto indices = BitCast(du, detail::IndicesFromBits128(d, mask_bits));
const auto compressed = BitCast(d, TableLookupBytes(BitCast(du, v), indices));
StoreU(compressed, d, unaligned);
detail::MaybeUnpoison(unaligned, count);
return count;
}
template <class T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, 1> SetAtOrAfterFirst(Mask128<T, 1> mask) {
return mask;
}
template <class T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, 2> SetAtOrAfterFirst(Mask128<T, 2> mask) {
const FixedTag<T, 2> d;
const auto vmask = VecFromMask(d, mask);
return MaskFromVec(Or(vmask, InterleaveLower(vmask, vmask)));
}
template <class T, size_t N, hwy::EnableIf<(N > 2)>* = nullptr, hwy::EnableIf<N * sizeof(T) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> SetAtOrAfterFirst(Mask128<T, N> mask) {
const Simd<T, N, 0> d;
const auto vmask = VecFromMask(d, mask);
const auto neg_vmask =
ResizeBitCast(d, Neg(ResizeBitCast(Full64<int64_t>(), vmask)));
return MaskFromVec(Or(vmask, neg_vmask));
}
template <class T, hwy::EnableIf<sizeof(T) != (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T> SetAtOrAfterFirst(Mask128<T> mask) {
const Full128<T> d;
const Repartition<int64_t, decltype(d)> di64;
const Repartition<float, decltype(d)> df32;
const Repartition<int32_t, decltype(d)> di32;
using VF = VFromD<decltype(df32)>;
auto vmask = BitCast(di64, VecFromMask(d, mask));
vmask = Or(vmask, Neg(vmask));
const auto vmask2 = BroadcastSignBit(
BitCast(di32, VF{_mm_shuffle_ps(Zero(df32).raw, BitCast(df32, vmask).raw,
(((
1
) << 6) | ((
1
) << 4) | ((
0
) << 2) | (
0
))
)}));
return MaskFromVec(BitCast(d, Or(vmask, BitCast(di64, vmask2))));
}
template <class T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> SetBeforeFirst(Mask128<T, N> mask) {
return Not(SetAtOrAfterFirst(mask));
}
template <class T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, 1> SetOnlyFirst(Mask128<T, 1> mask) {
return mask;
}
template <class T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, 2> SetOnlyFirst(Mask128<T, 2> mask) {
const FixedTag<T, 2> d;
const RebindToSigned<decltype(d)> di;
const auto vmask = BitCast(di, VecFromMask(d, mask));
const auto zero = Zero(di);
const auto vmask2 = VecFromMask(di, InterleaveLower(zero, vmask) == zero);
return MaskFromVec(BitCast(d, And(vmask, vmask2)));
}
template <class T, size_t N, hwy::EnableIf<(N > 2)>* = nullptr, hwy::EnableIf<N * sizeof(T) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> SetOnlyFirst(Mask128<T, N> mask) {
const Simd<T, N, 0> d;
const RebindToSigned<decltype(d)> di;
const auto vmask = ResizeBitCast(Full64<int64_t>(), VecFromMask(d, mask));
const auto only_first_vmask =
BitCast(d, Neg(ResizeBitCast(di, And(vmask, Neg(vmask)))));
return MaskFromVec(only_first_vmask);
}
template <class T, hwy::EnableIf<sizeof(T) != (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T> SetOnlyFirst(Mask128<T> mask) {
const Full128<T> d;
const RebindToSigned<decltype(d)> di;
const Repartition<int64_t, decltype(d)> di64;
const auto zero = Zero(di64);
const auto vmask = BitCast(di64, VecFromMask(d, mask));
const auto vmask2 = VecFromMask(di64, InterleaveLower(zero, vmask) == zero);
const auto only_first_vmask = Neg(BitCast(di, And(vmask, Neg(vmask))));
return MaskFromVec(BitCast(d, And(only_first_vmask, BitCast(di, vmask2))));
}
template <class T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, 1> SetAtOrBeforeFirst(Mask128<T, 1> ) {
const FixedTag<T, 1> d;
const RebindToSigned<decltype(d)> di;
using TI = MakeSigned<T>;
return RebindMask(d, MaskFromVec(Set(di, TI(-1))));
}
template <class T, size_t N, hwy::EnableIf<(N > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> SetAtOrBeforeFirst(Mask128<T, N> mask) {
const Simd<T, N, 0> d;
return SetBeforeFirst(MaskFromVec(ShiftLeftLanes<1>(VecFromMask(d, mask))));
}
namespace detail {
template <typename T>
inline __attribute__((always_inline)) Vec128<T, 1> SumOfLanes(Vec128<T, 1> v) {
return v;
}
template <typename T>
inline __attribute__((always_inline)) T ReduceSum(Vec128<T, 1> v) {
return GetLane(v);
}
template <typename T>
inline __attribute__((always_inline)) Vec128<T, 1> MinOfLanes(Vec128<T, 1> v) {
return v;
}
template <typename T>
inline __attribute__((always_inline)) Vec128<T, 1> MaxOfLanes(Vec128<T, 1> v) {
return v;
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, 2> SumOfLanes(Vec128<T, 2> v10) {
return v10 + Shuffle2301(v10);
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) T ReduceSum(Vec128<T, 2> v10) {
return GetLane(SumOfLanes(v10));
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, 2> MinOfLanes(Vec128<T, 2> v10) {
return Min(v10, Shuffle2301(v10));
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, 2> MaxOfLanes(Vec128<T, 2> v10) {
return Max(v10, Shuffle2301(v10));
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> SumOfLanes(Vec128<T> v3210) {
const Vec128<T> v1032 = Shuffle1032(v3210);
const Vec128<T> v31_20_31_20 = v3210 + v1032;
const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20);
return v20_31_20_31 + v31_20_31_20;
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) T ReduceSum(Vec128<T> v3210) {
return GetLane(SumOfLanes(v3210));
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> MinOfLanes(Vec128<T> v3210) {
const Vec128<T> v1032 = Shuffle1032(v3210);
const Vec128<T> v31_20_31_20 = Min(v3210, v1032);
const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20);
return Min(v20_31_20_31, v31_20_31_20);
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> MaxOfLanes(Vec128<T> v3210) {
const Vec128<T> v1032 = Shuffle1032(v3210);
const Vec128<T> v31_20_31_20 = Max(v3210, v1032);
const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20);
return Max(v20_31_20_31, v31_20_31_20);
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> SumOfLanes(Vec128<T> v10) {
const Vec128<T> v01 = Shuffle01(v10);
return v10 + v01;
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) T ReduceSum(Vec128<T> v10) {
return GetLane(SumOfLanes(v10));
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> MinOfLanes(Vec128<T> v10) {
const Vec128<T> v01 = Shuffle01(v10);
return Min(v10, v01);
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> MaxOfLanes(Vec128<T> v10) {
const Vec128<T> v01 = Shuffle01(v10);
return Max(v10, v01);
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) uint16_t ReduceSum(Vec128<uint16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto sum = ReduceSum(even + odd);
return static_cast<uint16_t>(sum);
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<uint16_t, N> SumOfLanes(Vec128<uint16_t, N> v) {
const DFromV<decltype(v)> d;
return Set(d, ReduceSum(v));
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) int16_t ReduceSum(Vec128<int16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = ShiftRight<16>(ShiftLeft<16>(BitCast(d32, v)));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto sum = ReduceSum(even + odd);
return static_cast<int16_t>(sum);
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int16_t, N> SumOfLanes(Vec128<int16_t, N> v) {
const DFromV<decltype(v)> d;
return Set(d, ReduceSum(v));
}
inline __attribute__((always_inline)) uint8_t ReduceSum(Vec64<uint8_t> v) {
return static_cast<uint8_t>(GetLane(SumsOf8(v)) & 0xFF);
}
inline __attribute__((always_inline)) Vec64<uint8_t> SumOfLanes(Vec64<uint8_t> v) {
const Full64<uint8_t> d;
return Set(d, ReduceSum(v));
}
inline __attribute__((always_inline)) uint8_t ReduceSum(Vec128<uint8_t> v) {
uint64_t sums = ReduceSum(SumsOf8(v));
return static_cast<uint8_t>(sums & 0xFF);
}
inline __attribute__((always_inline)) Vec128<uint8_t> SumOfLanes(Vec128<uint8_t> v) {
const DFromV<decltype(v)> d;
return Set(d, ReduceSum(v));
}
template <size_t N, hwy::EnableIf<(N * sizeof(int8_t) > 4)>* = nullptr>
inline __attribute__((always_inline)) int8_t ReduceSum(const Vec128<int8_t, N> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const auto is_neg = v < Zero(d);
const auto positive = SumsOf8(BitCast(du, IfThenZeroElse(is_neg, v)));
const auto negative = SumsOf8(BitCast(du, IfThenElseZero(is_neg, Abs(v))));
return static_cast<int8_t>(ReduceSum(positive - negative) & 0xFF);
}
template <size_t N, hwy::EnableIf<(N * sizeof(int8_t) > 4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int8_t, N> SumOfLanes(const Vec128<int8_t, N> v) {
const DFromV<decltype(v)> d;
return Set(d, ReduceSum(v));
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint8_t) > 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> MaxOfLanes(Vec128<uint8_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d16;
const RepartitionToWide<decltype(d16)> d32;
Vec128<uint8_t, N> vm = Max(v, Reverse2(d, v));
vm = Max(vm, BitCast(d, Reverse2(d16, BitCast(d16, vm))));
vm = Max(vm, BitCast(d, Reverse2(d32, BitCast(d32, vm))));
if (N > 8) {
const RepartitionToWide<decltype(d32)> d64;
vm = Max(vm, BitCast(d, Reverse2(d64, BitCast(d64, vm))));
}
return vm;
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint8_t) > 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> MinOfLanes(Vec128<uint8_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d16;
const RepartitionToWide<decltype(d16)> d32;
Vec128<uint8_t, N> vm = Min(v, Reverse2(d, v));
vm = Min(vm, BitCast(d, Reverse2(d16, BitCast(d16, vm))));
vm = Min(vm, BitCast(d, Reverse2(d32, BitCast(d32, vm))));
if (N > 8) {
const RepartitionToWide<decltype(d32)> d64;
vm = Min(vm, BitCast(d, Reverse2(d64, BitCast(d64, vm))));
}
return vm;
}
template <size_t N, hwy::EnableIf<(N * sizeof(int8_t) > 4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int8_t, N> MinOfLanes(Vec128<int8_t, N> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const auto mask = SignBit(du);
const auto vu = Xor(BitCast(du, v), mask);
return BitCast(d, Xor(MinOfLanes(vu), mask));
}
template <size_t N, hwy::EnableIf<(N * sizeof(int8_t) > 4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int8_t, N> MaxOfLanes(Vec128<int8_t, N> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const auto mask = SignBit(du);
const auto vu = Xor(BitCast(du, v), mask);
return BitCast(d, Xor(MaxOfLanes(vu), mask));
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<uint16_t, N> MinOfLanes(Vec128<uint16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto min = MinOfLanes(Min(even, odd));
return OddEven(BitCast(d, ShiftLeft<16>(min)), BitCast(d, min));
}
template <size_t N, hwy::EnableIf<(N * sizeof(int16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int16_t, N> MinOfLanes(Vec128<int16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = ShiftRight<16>(ShiftLeft<16>(BitCast(d32, v)));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto min = MinOfLanes(Min(even, odd));
return OddEven(BitCast(d, ShiftLeft<16>(min)), BitCast(d, min));
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<uint16_t, N> MaxOfLanes(Vec128<uint16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto min = MaxOfLanes(Max(even, odd));
return OddEven(BitCast(d, ShiftLeft<16>(min)), BitCast(d, min));
}
template <size_t N, hwy::EnableIf<(N * sizeof(int16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int16_t, N> MaxOfLanes(Vec128<int16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = ShiftRight<16>(ShiftLeft<16>(BitCast(d32, v)));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto min = MaxOfLanes(Max(even, odd));
return OddEven(BitCast(d, ShiftLeft<16>(min)), BitCast(d, min));
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SumOfLanes(D , VFromD<D> v) {
return detail::SumOfLanes(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) TFromD<D> ReduceSum(D , VFromD<D> v) {
return detail::ReduceSum(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> MinOfLanes(D , VFromD<D> v) {
return detail::MinOfLanes(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> MaxOfLanes(D , VFromD<D> v) {
return detail::MaxOfLanes(v);
}
namespace detail {
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Lt128Vec(const D d, const V a, const V b) {
static_assert(IsSame<TFromD<D>, uint64_t>(), "D must be u64");
const auto eqHL = Eq(a, b);
const V ltHL = VecFromMask(d, Lt(a, b));
const V ltLX = ShiftLeftLanes<1>(ltHL);
const V vecHx = IfThenElse(eqHL, ltLX, ltHL);
return InterleaveUpper(d, vecHx, vecHx);
}
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Eq128Vec(const D d, const V a, const V b) {
static_assert(IsSame<TFromD<D>, uint64_t>(), "D must be u64");
const auto eqHL = VecFromMask(d, Eq(a, b));
const auto eqLH = Reverse2(d, eqHL);
return And(eqHL, eqLH);
}
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Ne128Vec(const D d, const V a, const V b) {
static_assert(IsSame<TFromD<D>, uint64_t>(), "D must be u64");
const auto neHL = VecFromMask(d, Ne(a, b));
const auto neLH = Reverse2(d, neHL);
return Or(neHL, neLH);
}
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Lt128UpperVec(const D d, const V a, const V b) {
const V ltHL = VecFromMask(d, Lt(a, b));
return InterleaveUpper(d, ltHL, ltHL);
}
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Eq128UpperVec(const D d, const V a, const V b) {
const V eqHL = VecFromMask(d, Eq(a, b));
return InterleaveUpper(d, eqHL, eqHL);
}
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Ne128UpperVec(const D d, const V a, const V b) {
const V neHL = VecFromMask(d, Ne(a, b));
return InterleaveUpper(d, neHL, neHL);
}
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Lt128(D d, const V a, const V b) {
return MaskFromVec(detail::Lt128Vec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Eq128(D d, const V a, const V b) {
return MaskFromVec(detail::Eq128Vec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Ne128(D d, const V a, const V b) {
return MaskFromVec(detail::Ne128Vec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Lt128Upper(D d, const V a, const V b) {
return MaskFromVec(detail::Lt128UpperVec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Eq128Upper(D d, const V a, const V b) {
return MaskFromVec(detail::Eq128UpperVec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Ne128Upper(D d, const V a, const V b) {
return MaskFromVec(detail::Ne128UpperVec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Min128(D d, const V a, const V b) {
return IfVecThenElse(detail::Lt128Vec(d, a, b), a, b);
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Max128(D d, const V a, const V b) {
return IfVecThenElse(detail::Lt128Vec(d, b, a), a, b);
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Min128Upper(D d, const V a, const V b) {
return IfVecThenElse(detail::Lt128UpperVec(d, a, b), a, b);
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Max128Upper(D d, const V a, const V b) {
return IfVecThenElse(detail::Lt128UpperVec(d, b, a), a, b);
}
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC diagnostic pop
#pragma GCC push_options
#pragma GCC target "sse2,ssse3"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSSE3 {
template <class V>
using LaneType = decltype(GetLane(V()));
template <class D>
using Vec = decltype(Zero(D()));
template <class D>
using Mask = decltype(MaskFromVec(Zero(D())));
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Clamp(const V v, const V lo, const V hi) {
return Min(Max(lo, v), hi);
}
template <size_t kLanes, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> CombineShiftRightLanes(D d, VFromD<D> hi, VFromD<D> lo) {
constexpr size_t kBytes = kLanes * sizeof(TFromD<D>);
static_assert(kBytes < 16, "Shift count is per-block");
return CombineShiftRightBytes<kBytes>(d, hi, lo);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec<D> SignBit(D d) {
const RebindToUnsigned<decltype(d)> du;
return BitCast(d, Set(du, SignMask<TFromD<D>>()));
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec<D> NaN(D d) {
const RebindToSigned<D> di;
return BitCast(d, Set(di, LimitsMax<TFromD<decltype(di)>>()));
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec<D> Inf(D d) {
const RebindToUnsigned<D> du;
using T = TFromD<D>;
using TU = TFromD<decltype(du)>;
const TU max_x2 = static_cast<TU>(MaxExponentTimes2<T>());
return BitCast(d, Set(du, max_x2 >> 1));
}
namespace detail {
template <size_t kFromVectSize, size_t kToVectSize, class DTo, class DFrom,
hwy::EnableIf<(kToVectSize <= kFromVectSize)>* = nullptr>
inline __attribute__((always_inline)) VFromD<DTo> ZeroExtendResizeBitCast(
hwy::SizeTag<kFromVectSize> ,
hwy::SizeTag<kToVectSize> , DTo d_to, DFrom ,
VFromD<DFrom> v) {
return ResizeBitCast(d_to, v);
}
template <size_t kFromVectSize, size_t kToVectSize, class DTo, class DFrom,
hwy::EnableIf<(kToVectSize == kFromVectSize * 2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<DTo> ZeroExtendResizeBitCast(
hwy::SizeTag<kFromVectSize> ,
hwy::SizeTag<kToVectSize> , DTo d_to, DFrom d_from,
VFromD<DFrom> v) {
const Twice<decltype(d_from)> dt_from;
return BitCast(d_to, ZeroExtendVector(dt_from, v));
}
template <size_t kFromVectSize, size_t kToVectSize, class DTo, class DFrom,
hwy::EnableIf<(kToVectSize > kFromVectSize * 2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<DTo> ZeroExtendResizeBitCast(
hwy::SizeTag<kFromVectSize> ,
hwy::SizeTag<kToVectSize> , DTo d_to, DFrom ,
VFromD<DFrom> v) {
using TFrom = TFromD<DFrom>;
constexpr size_t kNumOfFromLanes = kFromVectSize / sizeof(TFrom);
const Repartition<TFrom, decltype(d_to)> d_resize_to;
return BitCast(d_to, IfThenElseZero(FirstN(d_resize_to, kNumOfFromLanes),
ResizeBitCast(d_resize_to, v)));
}
}
template <class DTo, class DFrom>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DTo> ZeroExtendResizeBitCast(DTo d_to, DFrom d_from,
VFromD<DFrom> v) {
return detail::ZeroExtendResizeBitCast(hwy::SizeTag<d_from.MaxBytes()>(),
hwy::SizeTag<d_to.MaxBytes()>(), d_to,
d_from, v);
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void SafeFillN(const size_t num, const T value, D d,
T* __restrict__ to) {
(void)d;
for (size_t i = 0; i < num; ++i) {
to[i] = value;
}
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void SafeCopyN(const size_t num, D d, const T* __restrict__ from,
T* __restrict__ to) {
(void)d;
for (size_t i = 0; i < num; ++i) {
to[i] = from[i];
}
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V BitwiseIfThenElse(V mask, V yes, V no) {
return Or(And(mask, yes), AndNot(mask, no));
}
template <class D, hwy::EnableIf<(D::kPrivateLanes > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved2(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1) {
const VFromD<D> A = LoadU(d, unaligned);
const VFromD<D> B = LoadU(d, unaligned + Lanes(d));
v0 = ConcatEven(d, B, A);
v1 = ConcatOdd(d, B, A);
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved2(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1) {
v0 = LoadU(d, unaligned + 0);
v1 = LoadU(d, unaligned + 1);
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) void LoadTransposedBlocks3(D d,
const TFromD<D>* __restrict__ unaligned,
VFromD<D>& A, VFromD<D>& B,
VFromD<D>& C) {
constexpr size_t kN = MaxLanes(d);
A = LoadU(d, unaligned + 0 * kN);
B = LoadU(d, unaligned + 1 * kN);
C = LoadU(d, unaligned + 2 * kN);
}
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (16)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
const RebindToUnsigned<decltype(d)> du;
using V = VFromD<D>;
V A;
V B;
V C;
detail::LoadTransposedBlocks3(d, unaligned, A, B, C);
constexpr uint8_t Z = 0x80;
alignas(16) static constexpr uint8_t kIdx_v0A[16] = {
0, 3, 6, 9, 12, 15, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0B[16] = {
Z, Z, Z, Z, Z, Z, 2, 5, 8, 11, 14, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 1, 4, 7, 10, 13};
alignas(16) static constexpr uint8_t kIdx_v1A[16] = {
1, 4, 7, 10, 13, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1B[16] = {
Z, Z, Z, Z, Z, 0, 3, 6, 9, 12, 15, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 2, 5, 8, 11, 14};
alignas(16) static constexpr uint8_t kIdx_v2A[16] = {
2, 5, 8, 11, 14, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2B[16] = {
Z, Z, Z, Z, Z, 1, 4, 7, 10, 13, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 0, 3, 6, 9, 12, 15};
const V v0L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v0A)));
const V v0M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v0B)));
const V v0U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v0C)));
const V v1L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v1A)));
const V v1M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v1B)));
const V v1U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v1C)));
const V v2L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v2A)));
const V v2M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v2B)));
const V v2U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v2C)));
v0 = Xor3(v0L, v0M, v0U);
v1 = Xor3(v1L, v1M, v1U);
v2 = Xor3(v2L, v2M, v2U);
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (8)>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
const RebindToUnsigned<decltype(d)> du;
using V = VFromD<D>;
V A;
V B;
V C;
detail::LoadTransposedBlocks3(d, unaligned, A, B, C);
constexpr uint8_t Z = 0x80;
alignas(16) static constexpr uint8_t kIdx_v0A[16] = {0, 3, 6, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0B[16] = {Z, Z, Z, 1, 4, 7, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0C[16] = {Z, Z, Z, Z, Z, Z, 2, 5};
alignas(16) static constexpr uint8_t kIdx_v1A[16] = {1, 4, 7, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1B[16] = {Z, Z, Z, 2, 5, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1C[16] = {Z, Z, Z, Z, Z, 0, 3, 6};
alignas(16) static constexpr uint8_t kIdx_v2A[16] = {2, 5, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2B[16] = {Z, Z, 0, 3, 6, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2C[16] = {Z, Z, Z, Z, Z, 1, 4, 7};
const V v0L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v0A)));
const V v0M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v0B)));
const V v0U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v0C)));
const V v1L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v1A)));
const V v1M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v1B)));
const V v1U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v1C)));
const V v2L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v2A)));
const V v2M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v2B)));
const V v2U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v2C)));
v0 = Xor3(v0L, v0M, v0U);
v1 = Xor3(v1L, v1M, v1U);
v2 = Xor3(v2L, v2M, v2U);
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (8)>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
const RebindToUnsigned<decltype(d)> du;
const Repartition<uint8_t, decltype(du)> du8;
using V = VFromD<D>;
V A;
V B;
V C;
detail::LoadTransposedBlocks3(d, unaligned, A, B, C);
constexpr uint8_t Z = 0x80;
alignas(16) static constexpr uint8_t kIdx_v0A[16] = {
0x00, 0x01, 0x06, 0x07, 0x0C, 0x0D, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0B[16] = {
Z, Z, Z, Z, Z, Z, 0x02, 0x03, 0x08, 0x09, 0x0E, 0x0F, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 0x04, 0x05, 0x0A, 0x0B};
alignas(16) static constexpr uint8_t kIdx_v1A[16] = {
0x02, 0x03, 0x08, 0x09, 0x0E, 0x0F, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1B[16] = {
Z, Z, Z, Z, Z, Z, 0x04, 0x05, 0x0A, 0x0B, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 0x00, 0x01, 0x06, 0x07, 0x0C, 0x0D};
alignas(16) static constexpr uint8_t kIdx_v2A[16] = {
0x04, 0x05, 0x0A, 0x0B, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2B[16] = {
Z, Z, Z, Z, 0x00, 0x01, 0x06, 0x07, 0x0C, 0x0D, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 0x02, 0x03, 0x08, 0x09, 0x0E, 0x0F};
const V v0L = TableLookupBytesOr0(A, BitCast(d, LoadDup128(du8, kIdx_v0A)));
const V v0M = TableLookupBytesOr0(B, BitCast(d, LoadDup128(du8, kIdx_v0B)));
const V v0U = TableLookupBytesOr0(C, BitCast(d, LoadDup128(du8, kIdx_v0C)));
const V v1L = TableLookupBytesOr0(A, BitCast(d, LoadDup128(du8, kIdx_v1A)));
const V v1M = TableLookupBytesOr0(B, BitCast(d, LoadDup128(du8, kIdx_v1B)));
const V v1U = TableLookupBytesOr0(C, BitCast(d, LoadDup128(du8, kIdx_v1C)));
const V v2L = TableLookupBytesOr0(A, BitCast(d, LoadDup128(du8, kIdx_v2A)));
const V v2M = TableLookupBytesOr0(B, BitCast(d, LoadDup128(du8, kIdx_v2B)));
const V v2U = TableLookupBytesOr0(C, BitCast(d, LoadDup128(du8, kIdx_v2C)));
v0 = Xor3(v0L, v0M, v0U);
v1 = Xor3(v1L, v1M, v1U);
v2 = Xor3(v2L, v2M, v2U);
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
using V = VFromD<D>;
V A;
V B;
V C;
detail::LoadTransposedBlocks3(d, unaligned, A, B, C);
const V vxx_02_03_xx = OddEven(C, B);
v0 = detail::ShuffleTwo1230(A, vxx_02_03_xx);
const V vxx_xx_10_11 = OddEven(A, B);
const V v12_13_xx_xx = OddEven(B, C);
v1 = detail::ShuffleTwo2301(vxx_xx_10_11, v12_13_xx_xx);
const V vxx_20_21_xx = OddEven(B, A);
v2 = detail::ShuffleTwo3012(vxx_20_21_xx, C);
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
VFromD<D> A;
VFromD<D> B;
VFromD<D> C;
detail::LoadTransposedBlocks3(d, unaligned, A, B, C);
v0 = OddEven(B, A);
v1 = CombineShiftRightBytes<sizeof(TFromD<D>)>(d, C, A);
v2 = OddEven(C, B);
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const T* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
v0 = LoadU(d, unaligned + 0);
v1 = LoadU(d, unaligned + 1);
v2 = LoadU(d, unaligned + 2);
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) void LoadTransposedBlocks4(D d,
const TFromD<D>* __restrict__ unaligned,
VFromD<D>& vA, VFromD<D>& vB,
VFromD<D>& vC, VFromD<D>& vD) {
constexpr size_t kN = MaxLanes(d);
vA = LoadU(d, unaligned + 0 * kN);
vB = LoadU(d, unaligned + 1 * kN);
vC = LoadU(d, unaligned + 2 * kN);
vD = LoadU(d, unaligned + 3 * kN);
}
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (16)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved4(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2,
VFromD<D>& v3) {
const Repartition<uint64_t, decltype(d)> d64;
using V64 = VFromD<decltype(d64)>;
using V = VFromD<D>;
V vA;
V vB;
V vC;
V vD;
detail::LoadTransposedBlocks4(d, unaligned, vA, vB, vC, vD);
const V v5140 = InterleaveLower(d, vA, vB);
const V vd9c8 = InterleaveLower(d, vC, vD);
const V v7362 = InterleaveUpper(d, vA, vB);
const V vfbea = InterleaveUpper(d, vC, vD);
const V v6420 = InterleaveLower(d, v5140, v7362);
const V veca8 = InterleaveLower(d, vd9c8, vfbea);
const V v7531 = InterleaveUpper(d, v5140, v7362);
const V vfdb9 = InterleaveUpper(d, vd9c8, vfbea);
const V64 v10L = BitCast(d64, InterleaveLower(d, v6420, v7531));
const V64 v10U = BitCast(d64, InterleaveLower(d, veca8, vfdb9));
const V64 v32L = BitCast(d64, InterleaveUpper(d, v6420, v7531));
const V64 v32U = BitCast(d64, InterleaveUpper(d, veca8, vfdb9));
v0 = BitCast(d, InterleaveLower(d64, v10L, v10U));
v1 = BitCast(d, InterleaveUpper(d64, v10L, v10U));
v2 = BitCast(d, InterleaveLower(d64, v32L, v32U));
v3 = BitCast(d, InterleaveUpper(d64, v32L, v32U));
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved4(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2,
VFromD<D>& v3) {
using TW = hwy::UnsignedFromSize<d.MaxBytes() == 8 ? 4 : 8>;
const Repartition<TW, decltype(d)> dw;
using VW = VFromD<decltype(dw)>;
VFromD<D> vA;
VFromD<D> vB;
VFromD<D> vC;
VFromD<D> vD;
detail::LoadTransposedBlocks4(d, unaligned, vA, vB, vC, vD);
const VFromD<D> va820 = InterleaveLower(d, vA, vB);
const VFromD<D> vec64 = InterleaveLower(d, vC, vD);
const VFromD<D> vb931 = InterleaveUpper(d, vA, vB);
const VFromD<D> vfd75 = InterleaveUpper(d, vC, vD);
const VW v10_b830 =
BitCast(dw, InterleaveLower(d, va820, vb931));
const VW v10_fc74 =
BitCast(dw, InterleaveLower(d, vec64, vfd75));
const VW v32_b830 =
BitCast(dw, InterleaveUpper(d, va820, vb931));
const VW v32_fc74 =
BitCast(dw, InterleaveUpper(d, vec64, vfd75));
v0 = BitCast(d, InterleaveLower(dw, v10_b830, v10_fc74));
v1 = BitCast(d, InterleaveUpper(dw, v10_b830, v10_fc74));
v2 = BitCast(d, InterleaveLower(dw, v32_b830, v32_fc74));
v3 = BitCast(d, InterleaveUpper(dw, v32_b830, v32_fc74));
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved4(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2,
VFromD<D>& v3) {
using V = VFromD<D>;
V vA;
V vB;
V vC;
V vD;
detail::LoadTransposedBlocks4(d, unaligned, vA, vB, vC, vD);
const V v10e = InterleaveLower(d, vA, vC);
const V v10o = InterleaveLower(d, vB, vD);
const V v32e = InterleaveUpper(d, vA, vC);
const V v32o = InterleaveUpper(d, vB, vD);
v0 = InterleaveLower(d, v10e, v10o);
v1 = InterleaveUpper(d, v10e, v10o);
v2 = InterleaveLower(d, v32e, v32o);
v3 = InterleaveUpper(d, v32e, v32o);
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved4(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2,
VFromD<D>& v3) {
VFromD<D> vA, vB, vC, vD;
detail::LoadTransposedBlocks4(d, unaligned, vA, vB, vC, vD);
v0 = InterleaveLower(d, vA, vC);
v1 = InterleaveUpper(d, vA, vC);
v2 = InterleaveLower(d, vB, vD);
v3 = InterleaveUpper(d, vB, vD);
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved4(D d, const T* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2,
VFromD<D>& v3) {
v0 = LoadU(d, unaligned + 0);
v1 = LoadU(d, unaligned + 1);
v2 = LoadU(d, unaligned + 2);
v3 = LoadU(d, unaligned + 3);
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) void StoreTransposedBlocks2(VFromD<D> A, VFromD<D> B, D d,
TFromD<D>* __restrict__ unaligned) {
constexpr size_t kN = MaxLanes(d);
StoreU(A, d, unaligned + 0 * kN);
StoreU(B, d, unaligned + 1 * kN);
}
}
template <class D, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved2(VFromD<D> v0, VFromD<D> v1, D d,
TFromD<D>* __restrict__ unaligned) {
const auto v10L = InterleaveLower(d, v0, v1);
const auto v10U = InterleaveUpper(d, v0, v1);
detail::StoreTransposedBlocks2(v10L, v10U, d, unaligned);
}
template <class V, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved2(V part0, V part1, D d,
TFromD<D>* __restrict__ unaligned) {
const Twice<decltype(d)> d2;
const auto v0 = ZeroExtendVector(d2, part0);
const auto v1 = ZeroExtendVector(d2, part1);
const auto v10 = InterleaveLower(d2, v0, v1);
StoreU(v10, d2, unaligned);
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) void StoreTransposedBlocks3(VFromD<D> A, VFromD<D> B, VFromD<D> C,
D d, TFromD<D>* __restrict__ unaligned) {
constexpr size_t kN = MaxLanes(d);
StoreU(A, d, unaligned + 0 * kN);
StoreU(B, d, unaligned + 1 * kN);
StoreU(C, d, unaligned + 2 * kN);
}
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
const auto k5 = Set(du, TU{5});
const auto k6 = Set(du, TU{6});
alignas(16) static constexpr uint8_t tbl_v0[16] = {
0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80, 0x80,
3, 0x80, 0x80, 4, 0x80, 0x80, 5};
alignas(16) static constexpr uint8_t tbl_v1[16] = {
0x80, 0, 0x80, 0x80, 1, 0x80,
0x80, 2, 0x80, 0x80, 3, 0x80, 0x80, 4, 0x80, 0x80};
const auto shuf_A0 = LoadDup128(du, tbl_v0);
const auto shuf_A1 = LoadDup128(du, tbl_v1);
const auto shuf_A2 = CombineShiftRightBytes<15>(du, shuf_A1, shuf_A1);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const VFromD<D> A = BitCast(d, A0 | A1 | A2);
const auto shuf_B0 = shuf_A2 + k6;
const auto shuf_B1 = shuf_A0 + k5;
const auto shuf_B2 = shuf_A1 + k5;
const auto B0 = TableLookupBytesOr0(v0, shuf_B0);
const auto B1 = TableLookupBytesOr0(v1, shuf_B1);
const auto B2 = TableLookupBytesOr0(v2, shuf_B2);
const VFromD<D> B = BitCast(d, B0 | B1 | B2);
const auto shuf_C0 = shuf_B2 + k6;
const auto shuf_C1 = shuf_B0 + k5;
const auto shuf_C2 = shuf_B1 + k5;
const auto C0 = TableLookupBytesOr0(v0, shuf_C0);
const auto C1 = TableLookupBytesOr0(v1, shuf_C1);
const auto C2 = TableLookupBytesOr0(v2, shuf_C2);
const VFromD<D> C = BitCast(d, C0 | C1 | C2);
detail::StoreTransposedBlocks3(A, B, C, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
const Repartition<uint8_t, decltype(d)> du8;
const auto k2 = Set(du8, uint8_t{2 * sizeof(TFromD<D>)});
const auto k3 = Set(du8, uint8_t{3 * sizeof(TFromD<D>)});
alignas(16) static constexpr uint8_t tbl_v1[16] = {
0x80, 0x80, 0, 1, 0x80, 0x80, 0x80, 0x80,
2, 3, 0x80, 0x80, 0x80, 0x80, 4, 5};
alignas(16) static constexpr uint8_t tbl_v2[16] = {
0x80, 0x80, 0x80, 0x80, 0, 1, 0x80, 0x80,
0x80, 0x80, 2, 3, 0x80, 0x80, 0x80, 0x80};
const auto shuf_A1 = LoadDup128(du8, tbl_v1);
const auto shuf_A0 = CombineShiftRightBytes<2>(du8, shuf_A1, shuf_A1);
const auto shuf_A2 = LoadDup128(du8, tbl_v2);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const VFromD<D> A = BitCast(d, A0 | A1 | A2);
const auto shuf_B0 = shuf_A1 + k3;
const auto shuf_B1 = shuf_A2 + k3;
const auto shuf_B2 = shuf_A0 + k2;
const auto B0 = TableLookupBytesOr0(v0, shuf_B0);
const auto B1 = TableLookupBytesOr0(v1, shuf_B1);
const auto B2 = TableLookupBytesOr0(v2, shuf_B2);
const VFromD<D> B = BitCast(d, B0 | B1 | B2);
const auto shuf_C0 = shuf_B1 + k3;
const auto shuf_C1 = shuf_B2 + k3;
const auto shuf_C2 = shuf_B0 + k2;
const auto C0 = TableLookupBytesOr0(v0, shuf_C0);
const auto C1 = TableLookupBytesOr0(v1, shuf_C1);
const auto C2 = TableLookupBytesOr0(v2, shuf_C2);
const VFromD<D> C = BitCast(d, C0 | C1 | C2);
detail::StoreTransposedBlocks3(A, B, C, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
const RepartitionToWide<decltype(d)> dw;
const VFromD<D> v10_v00 = InterleaveLower(d, v0, v1);
const VFromD<D> v01_v20 = OddEven(v0, v2);
const VFromD<D> A = BitCast(
d, InterleaveLower(dw, BitCast(dw, v10_v00), BitCast(dw, v01_v20)));
const VFromD<D> v1_321 = ShiftRightLanes<1>(d, v1);
const VFromD<D> v0_32 = ShiftRightLanes<2>(d, v0);
const VFromD<D> v21_v11 = OddEven(v2, v1_321);
const VFromD<D> v12_v02 = OddEven(v1_321, v0_32);
const VFromD<D> B = BitCast(
d, InterleaveLower(dw, BitCast(dw, v21_v11), BitCast(dw, v12_v02)));
const VFromD<D> v23_v13 = OddEven(v2, v1_321);
const VFromD<D> v03_v22 = OddEven(v0, v2);
const VFromD<D> C = BitCast(
d, InterleaveUpper(dw, BitCast(dw, v03_v22), BitCast(dw, v23_v13)));
detail::StoreTransposedBlocks3(A, B, C, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
const VFromD<D> A = InterleaveLower(d, v0, v1);
const VFromD<D> B = OddEven(v0, v2);
const VFromD<D> C = InterleaveUpper(d, v1, v2);
detail::StoreTransposedBlocks3(A, B, C, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, D d,
TFromD<D>* __restrict__ unaligned) {
constexpr size_t kFullN = 16 / sizeof(TFromD<D>);
const Full128<uint8_t> du;
const Full128<TFromD<D>> d_full;
const auto k5 = Set(du, uint8_t{5});
const auto k6 = Set(du, uint8_t{6});
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
alignas(16) static constexpr uint8_t tbl_v0[16] = {
0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80, 0x80,
3, 0x80, 0x80, 4, 0x80, 0x80, 5};
alignas(16) static constexpr uint8_t tbl_v1[16] = {
0x80, 0, 0x80, 0x80, 1, 0x80,
0x80, 2, 0x80, 0x80, 3, 0x80, 0x80, 4, 0x80, 0x80};
const auto shuf_A0 = Load(du, tbl_v0);
const auto shuf_A1 = Load(du, tbl_v1);
const auto shuf_A2 = CombineShiftRightBytes<15>(du, shuf_A1, shuf_A1);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const auto A = BitCast(d_full, A0 | A1 | A2);
StoreU(A, d_full, unaligned + 0 * kFullN);
const auto shuf_B0 = shuf_A2 + k6;
const auto shuf_B1 = shuf_A0 + k5;
const auto shuf_B2 = shuf_A1 + k5;
const auto B0 = TableLookupBytesOr0(v0, shuf_B0);
const auto B1 = TableLookupBytesOr0(v1, shuf_B1);
const auto B2 = TableLookupBytesOr0(v2, shuf_B2);
const VFromD<D> B{BitCast(d_full, B0 | B1 | B2).raw};
StoreU(B, d, unaligned + 1 * kFullN);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, D dh,
TFromD<D>* __restrict__ unaligned) {
const Twice<D> d_full;
const Full128<uint8_t> du8;
const auto k2 = Set(du8, uint8_t{2 * sizeof(TFromD<D>)});
const auto k3 = Set(du8, uint8_t{3 * sizeof(TFromD<D>)});
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
alignas(16) static constexpr uint8_t tbl_v1[16] = {
0x80, 0x80, 0, 1, 0x80, 0x80, 0x80, 0x80,
2, 3, 0x80, 0x80, 0x80, 0x80, 4, 5};
alignas(16) static constexpr uint8_t tbl_v2[16] = {
0x80, 0x80, 0x80, 0x80, 0, 1, 0x80, 0x80,
0x80, 0x80, 2, 3, 0x80, 0x80, 0x80, 0x80};
const auto shuf_A1 = Load(du8, tbl_v1);
const auto shuf_A0 = CombineShiftRightBytes<2>(du8, shuf_A1, shuf_A1);
const auto shuf_A2 = Load(du8, tbl_v2);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const VFromD<decltype(d_full)> A = BitCast(d_full, A0 | A1 | A2);
StoreU(A, d_full, unaligned);
const auto shuf_B0 = shuf_A1 + k3;
const auto shuf_B1 = shuf_A2 + k3;
const auto shuf_B2 = shuf_A0 + k2;
const auto B0 = TableLookupBytesOr0(v0, shuf_B0);
const auto B1 = TableLookupBytesOr0(v1, shuf_B1);
const auto B2 = TableLookupBytesOr0(v2, shuf_B2);
const VFromD<decltype(d_full)> B = BitCast(d_full, B0 | B1 | B2);
StoreU(VFromD<D>{B.raw}, dh, unaligned + MaxLanes(d_full));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
const VFromD<D> v10_v00 = InterleaveLower(d, v0, v1);
const VFromD<D> v01_v20 = OddEven(v0, v2);
const VFromD<D> v21_v11 = InterleaveUpper(d, v1, v2);
constexpr size_t kN = MaxLanes(d);
StoreU(v10_v00, d, unaligned + 0 * kN);
StoreU(v01_v20, d, unaligned + 1 * kN);
StoreU(v21_v11, d, unaligned + 2 * kN);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr,
hwy::EnableIf<(D::kPrivateLanes > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, D d,
TFromD<D>* __restrict__ unaligned) {
const Full128<uint8_t> du;
const Full128<TFromD<D>> d_full;
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
alignas(16) static constexpr uint8_t tbl_v0[16] = {
0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80,
0x80, 3, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};
const auto shuf_A0 = Load(du, tbl_v0);
const auto shuf_A1 = CombineShiftRightBytes<15>(du, shuf_A0, shuf_A0);
const auto shuf_A2 = CombineShiftRightBytes<14>(du, shuf_A0, shuf_A0);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const VFromD<decltype(d_full)> A = BitCast(d_full, A0 | A1 | A2);
alignas(16) TFromD<D> buf[MaxLanes(d_full)];
StoreU(A, d_full, buf);
CopyBytes<d.MaxBytes() * 3>(buf, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, D d,
TFromD<D>* __restrict__ unaligned) {
const Full128<uint8_t> du8;
const Full128<TFromD<D>> d_full;
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
alignas(16) static constexpr uint8_t tbl_v2[16] = {
0x80, 0x80, 0x80, 0x80, 0, 1, 0x80, 0x80,
0x80, 0x80, 2, 3, 0x80, 0x80, 0x80, 0x80};
const auto shuf_A2 =
Load(du8, tbl_v2);
const auto shuf_A1 =
CombineShiftRightBytes<2>(du8, shuf_A2, shuf_A2);
const auto shuf_A0 =
CombineShiftRightBytes<4>(du8, shuf_A2, shuf_A2);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const auto A = BitCast(d_full, A0 | A1 | A2);
alignas(16) TFromD<D> buf[MaxLanes(d_full)];
StoreU(A, d_full, buf);
CopyBytes<d.MaxBytes() * 3>(buf, unaligned);
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
StoreU(v0, d, unaligned + 0);
StoreU(v1, d, unaligned + 1);
StoreU(v2, d, unaligned + 2);
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) void StoreTransposedBlocks4(VFromD<D> vA, VFromD<D> vB, VFromD<D> vC,
VFromD<D> vD, D d,
TFromD<D>* __restrict__ unaligned) {
constexpr size_t kN = MaxLanes(d);
StoreU(vA, d, unaligned + 0 * kN);
StoreU(vB, d, unaligned + 1 * kN);
StoreU(vC, d, unaligned + 2 * kN);
StoreU(vD, d, unaligned + 3 * kN);
}
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) != (8)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved4(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2,
VFromD<D> v3, D d,
TFromD<D>* __restrict__ unaligned) {
const RepartitionToWide<decltype(d)> dw;
const auto v10L = ZipLower(dw, v0, v1);
const auto v32L = ZipLower(dw, v2, v3);
const auto v10U = ZipUpper(dw, v0, v1);
const auto v32U = ZipUpper(dw, v2, v3);
const VFromD<D> vA = BitCast(d, InterleaveLower(dw, v10L, v32L));
const VFromD<D> vB = BitCast(d, InterleaveUpper(dw, v10L, v32L));
const VFromD<D> vC = BitCast(d, InterleaveLower(dw, v10U, v32U));
const VFromD<D> vD = BitCast(d, InterleaveUpper(dw, v10U, v32U));
detail::StoreTransposedBlocks4(vA, vB, vC, vD, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved4(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2,
VFromD<D> v3, D d,
TFromD<D>* __restrict__ unaligned) {
const VFromD<D> vA = InterleaveLower(d, v0, v1);
const VFromD<D> vB = InterleaveLower(d, v2, v3);
const VFromD<D> vC = InterleaveUpper(d, v0, v1);
const VFromD<D> vD = InterleaveUpper(d, v2, v3);
detail::StoreTransposedBlocks4(vA, vB, vC, vD, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) != (8)>* = nullptr, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved4(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, VFromD<D> part3, D ,
TFromD<D>* __restrict__ unaligned) {
const Full128<TFromD<D>> d_full;
const RepartitionToWide<decltype(d_full)> dw;
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
const VFromD<decltype(d_full)> v3{part3.raw};
const auto v10 = ZipLower(dw, v0, v1);
const auto v32 = ZipLower(dw, v2, v3);
const auto A = BitCast(d_full, InterleaveLower(dw, v10, v32));
const auto B = BitCast(d_full, InterleaveUpper(dw, v10, v32));
StoreU(A, d_full, unaligned);
StoreU(B, d_full, unaligned + MaxLanes(d_full));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved4(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, VFromD<D> part3, D ,
TFromD<D>* __restrict__ unaligned) {
const Full128<TFromD<D>> d_full;
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
const VFromD<decltype(d_full)> v3{part3.raw};
const auto A = InterleaveLower(d_full, v0, v1);
const auto B = InterleaveLower(d_full, v2, v3);
StoreU(A, d_full, unaligned);
StoreU(B, d_full, unaligned + MaxLanes(d_full));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved4(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, VFromD<D> part3, D d,
TFromD<D>* __restrict__ unaligned) {
const Full128<TFromD<D>> d_full;
const RepartitionToWide<decltype(d_full)> dw;
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
const VFromD<decltype(d_full)> v3{part3.raw};
const auto v10 = ZipLower(dw, v0, v1);
const auto v32 = ZipLower(dw, v2, v3);
const auto v3210 = BitCast(d_full, InterleaveLower(dw, v10, v32));
alignas(16) TFromD<D> buf[MaxLanes(d_full)];
StoreU(v3210, d_full, buf);
CopyBytes<d.MaxBytes() * 4>(buf, unaligned);
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AbsDiff(V a, V b) {
return Sub(Max(a, b), Min(a, b));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedAdd(V a, V b) {
const DFromV<decltype(a)> d;
const auto sum = Add(a, b);
const auto overflow_mask =
MaskFromVec(BroadcastSignBit(AndNot(Xor(a, b), Xor(a, sum))));
const auto overflow_result =
Xor(BroadcastSignBit(a), Set(d, LimitsMax<int32_t>()));
return IfThenElse(overflow_mask, overflow_result, sum);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedSub(V a, V b) {
const DFromV<decltype(a)> d;
const auto diff = Sub(a, b);
const auto overflow_mask =
MaskFromVec(BroadcastSignBit(And(Xor(a, b), Xor(a, diff))));
const auto overflow_result =
Xor(BroadcastSignBit(a), Set(d, LimitsMax<int32_t>()));
return IfThenElse(overflow_mask, overflow_result, diff);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedAdd(V a, V b) {
const DFromV<decltype(a)> d;
const auto sum = Add(a, b);
const auto overflow_mask =
MaskFromVec(BroadcastSignBit(AndNot(Xor(a, b), Xor(a, sum))));
const auto overflow_result =
Xor(BroadcastSignBit(a), Set(d, LimitsMax<int64_t>()));
return IfThenElse(overflow_mask, overflow_result, sum);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedSub(V a, V b) {
const DFromV<decltype(a)> d;
const auto diff = Sub(a, b);
const auto overflow_mask =
MaskFromVec(BroadcastSignBit(And(Xor(a, b), Xor(a, diff))));
const auto overflow_result =
Xor(BroadcastSignBit(a), Set(d, LimitsMax<int64_t>()));
return IfThenElse(overflow_mask, overflow_result, diff);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedAdd(V a, V b) {
return Add(a, Min(b, Not(a)));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedSub(V a, V b) {
return Sub(a, Min(a, b));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedAdd(V a, V b) {
return Add(a, Min(b, Not(a)));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedSub(V a, V b) {
return Sub(a, Min(a, b));
}
template <class DN, hwy::EnableIf<IsSigned<TFromD<DN> >() && !IsFloat<TFromD<DN> >() && !IsSpecialFloat<TFromD<DN> >()>* = nullptr, class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
class V2 = VFromD<Rebind<TFromV<V>, DN>>,
hwy::EnableIf<(sizeof(TFromD<DN>) < sizeof(TFromV<V>))>* = nullptr,
hwy::EnableIf<(DFromV<V>::kPrivateLanes == DFromV<V2>::kPrivateLanes)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DN> DemoteTo(DN dn, V v) {
const DFromV<decltype(v)> d;
const RebindToSigned<decltype(d)> di;
const RebindToUnsigned<decltype(dn)> dn_u;
const auto i2i_demote_result = DemoteTo(dn, BitCast(di, v));
const auto max_signed_val = Set(dn, hwy::HighestValue<TFromD<DN>>());
return BitCast(
dn, Min(BitCast(dn_u, i2i_demote_result), BitCast(dn_u, max_signed_val)));
}
template <class DN, hwy::EnableIf<IsSigned<TFromD<DN> >() && !IsFloat<TFromD<DN> >() && !IsSpecialFloat<TFromD<DN> >()>* = nullptr, class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
class V2 = VFromD<Repartition<TFromV<V>, DN>>,
hwy::EnableIf<sizeof(TFromV<V>) == (sizeof(TFromD<DN>) * 2)>* = nullptr,
hwy::EnableIf<(DFromV<V>::kPrivateLanes == DFromV<V2>::kPrivateLanes)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DN> ReorderDemote2To(DN dn, V a, V b) {
const DFromV<decltype(a)> d;
const RebindToSigned<decltype(d)> di;
const RebindToUnsigned<decltype(dn)> dn_u;
const auto i2i_demote_result =
ReorderDemote2To(dn, BitCast(di, a), BitCast(di, b));
const auto max_signed_val = Set(dn, hwy::HighestValue<TFromD<DN>>());
return BitCast(
dn, Min(BitCast(dn_u, i2i_demote_result), BitCast(dn_u, max_signed_val)));
}
template <class DN, hwy::EnableIf<!IsSigned<TFromD<DN> >()>* = nullptr, class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
hwy::EnableIf<sizeof(TFromV<V>) == (sizeof(TFromD<DN>) * 2)>* = nullptr,
hwy::EnableIf<(DFromV<VFromD<DN>>::kPrivateLanes == DFromV<V>::kPrivateLanes * 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DN> OrderedTruncate2To(DN dn, V a, V b) {
return ConcatEven(dn, BitCast(dn, b), BitCast(dn, a));
}
namespace detail {
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const RebindToFloat<decltype(d)> df;
const RebindToSigned<decltype(d)> di;
const Repartition<int16_t, decltype(d)> di16;
const auto f32_bits = BitCast(d, ConvertTo(df, BitCast(di, v)));
return BitCast(d, Min(BitCast(di16, ShiftRight<23>(f32_bits)),
BitCast(di16, Set(d, 158))));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint32_t>()>* = nullptr>
inline __attribute__((always_inline)) V I32RangeU32ToF32BiasedExp(V v) {
const DFromV<decltype(v)> d;
const RebindToFloat<decltype(d)> df;
const RebindToSigned<decltype(d)> d_src;
const auto f32_bits = BitCast(d, ConvertTo(df, BitCast(d_src, v)));
return ShiftRight<23>(f32_bits);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes <= 16 / 4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Rebind<uint32_t, decltype(d)> du32;
const auto f32_biased_exp_as_u32 =
I32RangeU32ToF32BiasedExp(PromoteTo(du32, v));
return TruncateTo(d, f32_biased_exp_as_u32);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 16 / 4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Half<decltype(d)> dh;
const Rebind<uint32_t, decltype(dh)> du32;
const auto lo_u32 = PromoteTo(du32, LowerHalf(dh, v));
const auto hi_u32 = PromoteTo(du32, UpperHalf(dh, v));
const auto lo_f32_biased_exp_as_u32 = I32RangeU32ToF32BiasedExp(lo_u32);
const auto hi_f32_biased_exp_as_u32 = I32RangeU32ToF32BiasedExp(hi_u32);
const RebindToSigned<decltype(du32)> di32;
const RebindToSigned<decltype(d)> di;
return BitCast(d,
OrderedDemote2To(di, BitCast(di32, lo_f32_biased_exp_as_u32),
BitCast(di32, hi_f32_biased_exp_as_u32)));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes <= 16 / 4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Rebind<uint32_t, decltype(d)> du32;
const auto f32_biased_exp_as_u32 =
I32RangeU32ToF32BiasedExp(PromoteTo(du32, v));
return U8FromU32(f32_biased_exp_as_u32);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 16 / 4)>* = nullptr,
hwy::EnableIf<(D::kPrivateLanes <= 16 / 2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Half<decltype(d)> dh;
const Rebind<uint32_t, decltype(dh)> du32;
const Repartition<uint16_t, decltype(du32)> du16;
const auto lo_u32 = PromoteTo(du32, LowerHalf(dh, v));
const auto hi_u32 = PromoteTo(du32, UpperHalf(dh, v));
const auto lo_f32_biased_exp_as_u32 = I32RangeU32ToF32BiasedExp(lo_u32);
const auto hi_f32_biased_exp_as_u32 = I32RangeU32ToF32BiasedExp(hi_u32);
const RebindToSigned<decltype(du32)> di32;
const RebindToSigned<decltype(du16)> di16;
const auto f32_biased_exp_as_i16 =
OrderedDemote2To(di16, BitCast(di32, lo_f32_biased_exp_as_u32),
BitCast(di32, hi_f32_biased_exp_as_u32));
return DemoteTo(d, f32_biased_exp_as_i16);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 16 / 2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Half<decltype(d)> dh;
const Half<decltype(dh)> dq;
const Rebind<uint32_t, decltype(dq)> du32;
const Repartition<uint16_t, decltype(du32)> du16;
const auto lo_half = LowerHalf(dh, v);
const auto hi_half = UpperHalf(dh, v);
const auto u32_q0 = PromoteTo(du32, LowerHalf(dq, lo_half));
const auto u32_q1 = PromoteTo(du32, UpperHalf(dq, lo_half));
const auto u32_q2 = PromoteTo(du32, LowerHalf(dq, hi_half));
const auto u32_q3 = PromoteTo(du32, UpperHalf(dq, hi_half));
const auto f32_biased_exp_as_u32_q0 = I32RangeU32ToF32BiasedExp(u32_q0);
const auto f32_biased_exp_as_u32_q1 = I32RangeU32ToF32BiasedExp(u32_q1);
const auto f32_biased_exp_as_u32_q2 = I32RangeU32ToF32BiasedExp(u32_q2);
const auto f32_biased_exp_as_u32_q3 = I32RangeU32ToF32BiasedExp(u32_q3);
const RebindToSigned<decltype(du32)> di32;
const RebindToSigned<decltype(du16)> di16;
const auto lo_f32_biased_exp_as_i16 =
OrderedDemote2To(di16, BitCast(di32, f32_biased_exp_as_u32_q0),
BitCast(di32, f32_biased_exp_as_u32_q1));
const auto hi_f32_biased_exp_as_i16 =
OrderedDemote2To(di16, BitCast(di32, f32_biased_exp_as_u32_q2),
BitCast(di32, f32_biased_exp_as_u32_q3));
return OrderedDemote2To(d, lo_f32_biased_exp_as_i16,
hi_f32_biased_exp_as_i16);
}
template <class D>
using F32ExpLzcntMinMaxRepartition = Repartition<uint8_t, D>;
template <class V>
using F32ExpLzcntMinMaxCmpV = VFromD<F32ExpLzcntMinMaxRepartition<DFromV<V>>>;
template <class V>
inline __attribute__((always_inline)) F32ExpLzcntMinMaxCmpV<V> F32ExpLzcntMinMaxBitCast(V v) {
const DFromV<decltype(v)> d;
const F32ExpLzcntMinMaxRepartition<decltype(d)> d2;
return BitCast(d2, v);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Repartition<uint32_t, decltype(d)> du32;
const auto f32_biased_exp = UIntToF32BiasedExp(du32, BitCast(du32, v));
const auto f32_biased_exp_adj =
IfThenZeroElse(Eq(f32_biased_exp, Zero(du32)),
BitCast(du32, Set(d, 0x0000002000000000u)));
const auto adj_f32_biased_exp = Add(f32_biased_exp, f32_biased_exp_adj);
return ShiftRight<32>(BitCast(
d, Max(F32ExpLzcntMinMaxBitCast(adj_f32_biased_exp),
F32ExpLzcntMinMaxBitCast(Reverse2(du32, adj_f32_biased_exp)))));
}
template <class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr>
inline __attribute__((always_inline)) V UIntToF32BiasedExp(V v) {
const DFromV<decltype(v)> d;
return UIntToF32BiasedExp(d, v);
}
template <class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2))) != 0>* = nullptr>
inline __attribute__((always_inline)) V NormalizeForUIntTruncConvToF32(V v) {
return v;
}
template <class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 4) | (1 << 8))) != 0>* = nullptr>
inline __attribute__((always_inline)) V NormalizeForUIntTruncConvToF32(V v) {
return AndNot(ShiftRight<24>(v), v);
}
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V HighestSetBitIndex(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
const auto f32_biased_exp = detail::UIntToF32BiasedExp(
detail::NormalizeForUIntTruncConvToF32(BitCast(du, v)));
return BitCast(d, Sub(f32_biased_exp, Set(du, TU{127})));
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V LeadingZeroCount(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
constexpr TU kNumOfBitsInT{sizeof(TU) * 8};
const auto f32_biased_exp = detail::UIntToF32BiasedExp(
detail::NormalizeForUIntTruncConvToF32(BitCast(du, v)));
const auto lz_count = Sub(Set(du, TU{kNumOfBitsInT + 126}), f32_biased_exp);
return BitCast(d,
Min(detail::F32ExpLzcntMinMaxBitCast(lz_count),
detail::F32ExpLzcntMinMaxBitCast(Set(du, kNumOfBitsInT))));
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V TrailingZeroCount(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const RebindToSigned<decltype(d)> di;
using TU = TFromD<decltype(du)>;
const auto vi = BitCast(di, v);
const auto lowest_bit = BitCast(du, And(vi, Neg(vi)));
constexpr TU kNumOfBitsInT{sizeof(TU) * 8};
const auto f32_biased_exp = detail::UIntToF32BiasedExp(lowest_bit);
const auto tz_count = Sub(f32_biased_exp, Set(du, TU{127}));
return BitCast(d,
Min(detail::F32ExpLzcntMinMaxBitCast(tz_count),
detail::F32ExpLzcntMinMaxBitCast(Set(du, kNumOfBitsInT))));
}
namespace detail {
template <class V>
inline __attribute__((always_inline)) V SubBytesMulInverseAndAffineLookup(V state, V affine_tblL,
V affine_tblU) {
const DFromV<V> du;
const auto mask = Set(du, uint8_t{0xF});
{
alignas(16) static constexpr uint8_t basisL[16] = {
0x00, 0x70, 0x2A, 0x5A, 0x98, 0xE8, 0xB2, 0xC2,
0x08, 0x78, 0x22, 0x52, 0x90, 0xE0, 0xBA, 0xCA};
alignas(16) static constexpr uint8_t basisU[16] = {
0x00, 0x4D, 0x7C, 0x31, 0x7D, 0x30, 0x01, 0x4C,
0x81, 0xCC, 0xFD, 0xB0, 0xFC, 0xB1, 0x80, 0xCD};
const auto sL = And(state, mask);
const auto sU = ShiftRight<4>(state);
const auto gf4L = TableLookupBytes(LoadDup128(du, basisL), sL);
const auto gf4U = TableLookupBytes(LoadDup128(du, basisU), sU);
state = Xor(gf4L, gf4U);
}
alignas(16) static constexpr uint8_t kZetaInv[16] = {
0x80, 7, 11, 15, 6, 10, 4, 1, 9, 8, 5, 2, 12, 14, 13, 3};
alignas(16) static constexpr uint8_t kInv[16] = {
0x80, 1, 8, 13, 15, 6, 5, 14, 2, 12, 11, 10, 9, 3, 7, 4};
const auto tbl = LoadDup128(du, kInv);
const auto sL = And(state, mask);
const auto sU = ShiftRight<4>(state);
const auto sX = Xor(sU, sL);
const auto invL = TableLookupBytes(LoadDup128(du, kZetaInv), sL);
const auto invU = TableLookupBytes(tbl, sU);
const auto invX = TableLookupBytes(tbl, sX);
const auto outL = Xor(sX, TableLookupBytesOr0(tbl, Xor(invL, invU)));
const auto outU = Xor(sU, TableLookupBytesOr0(tbl, Xor(invL, invX)));
const auto affL = TableLookupBytesOr0(affine_tblL, outL);
const auto affU = TableLookupBytesOr0(affine_tblU, outU);
return Xor(affL, affU);
}
template <class V>
inline __attribute__((always_inline)) V SubBytes(V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t kAffineL[16] = {
0x00, 0xC7, 0xBD, 0x6F, 0x17, 0x6D, 0xD2, 0xD0,
0x78, 0xA8, 0x02, 0xC5, 0x7A, 0xBF, 0xAA, 0x15};
alignas(16) static constexpr uint8_t kAffineU[16] = {
0x00, 0x6A, 0xBB, 0x5F, 0xA5, 0x74, 0xE4, 0xCF,
0xFA, 0x35, 0x2B, 0x41, 0xD1, 0x90, 0x1E, 0x8E};
return Xor(SubBytesMulInverseAndAffineLookup(state, LoadDup128(du, kAffineL),
LoadDup128(du, kAffineU)),
Set(du, uint8_t{0x63}));
}
template <class V>
inline __attribute__((always_inline)) V InvSubBytes(V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t kGF2P4InvToGF2P8InvL[16]{
0x00, 0x40, 0xF9, 0x7E, 0x53, 0xEA, 0x87, 0x13,
0x2D, 0x3E, 0x94, 0xD4, 0xB9, 0x6D, 0xAA, 0xC7};
alignas(16) static constexpr uint8_t kGF2P4InvToGF2P8InvU[16]{
0x00, 0x1D, 0x44, 0x93, 0x0F, 0x56, 0xD7, 0x12,
0x9C, 0x8E, 0xC5, 0xD8, 0x59, 0x81, 0x4B, 0xCA};
const auto b = Xor(Xor3(Or(ShiftLeft<1>(state), ShiftRight<7>(state)),
Or(ShiftLeft<3>(state), ShiftRight<5>(state)),
Or(ShiftLeft<6>(state), ShiftRight<2>(state))),
Set(du, uint8_t{0x05}));
return SubBytesMulInverseAndAffineLookup(
b, LoadDup128(du, kGF2P4InvToGF2P8InvL),
LoadDup128(du, kGF2P4InvToGF2P8InvU));
}
}
namespace detail {
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ShiftRows(const V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t kShiftRow[16] = {
0, 5, 10, 15,
4, 9, 14, 3,
8, 13, 2, 7,
12, 1, 6, 11};
const auto shift_row = LoadDup128(du, kShiftRow);
return TableLookupBytes(state, shift_row);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V InvShiftRows(const V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t kShiftRow[16] = {
0, 13, 10, 7,
4, 1, 14, 11,
8, 5, 2, 15,
12, 9, 6, 3};
const auto shift_row = LoadDup128(du, kShiftRow);
return TableLookupBytes(state, shift_row);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V GF2P8Mod11BMulBy2(V v) {
const DFromV<V> du;
const RebindToSigned<decltype(du)> di;
const auto msb = Lt(BitCast(di, v), Zero(di));
const auto overflow = BitCast(du, IfThenElseZero(msb, Set(di, int8_t{0x1B})));
return Xor(Add(v, v), overflow);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V MixColumns(const V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t k2301[16] = {
2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13};
alignas(16) static constexpr uint8_t k1230[16] = {
1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12};
const auto d = GF2P8Mod11BMulBy2(state);
const auto s2301 = TableLookupBytes(state, LoadDup128(du, k2301));
const auto d_s2301 = Xor(d, s2301);
const auto t_s2301 = Xor(state, d_s2301);
const auto t1230_s3012 = TableLookupBytes(t_s2301, LoadDup128(du, k1230));
return Xor(d_s2301, t1230_s3012);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V InvMixColumns(const V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t k2301[16] = {
2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13};
alignas(16) static constexpr uint8_t k1230[16] = {
1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12};
const auto v1230 = LoadDup128(du, k1230);
const auto sx2 = GF2P8Mod11BMulBy2(state);
const auto sx4 = GF2P8Mod11BMulBy2(sx2);
const auto sx8 = GF2P8Mod11BMulBy2(sx4);
const auto sx9 = Xor(sx8, state);
const auto sx11 = Xor(sx9, sx2);
const auto sx13 = Xor(sx9, sx4);
const auto sx14 = Xor3(sx8, sx4, sx2);
const auto sx13_0123_sx9_1230 = Xor(sx13, TableLookupBytes(sx9, v1230));
const auto sx14_0123_sx11_1230 = Xor(sx14, TableLookupBytes(sx11, v1230));
const auto sx13_2301_sx9_3012 =
TableLookupBytes(sx13_0123_sx9_1230, LoadDup128(du, k2301));
return Xor(sx14_0123_sx11_1230, sx13_2301_sx9_3012);
}
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESRound(V state, const V round_key) {
state = detail::SubBytes(state);
state = detail::ShiftRows(state);
state = detail::MixColumns(state);
state = Xor(state, round_key);
return state;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESLastRound(V state, const V round_key) {
state = detail::SubBytes(state);
state = detail::ShiftRows(state);
state = Xor(state, round_key);
return state;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESInvMixColumns(V state) {
return detail::InvMixColumns(state);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESRoundInv(V state, const V round_key) {
state = detail::InvSubBytes(state);
state = detail::InvShiftRows(state);
state = detail::InvMixColumns(state);
state = Xor(state, round_key);
return state;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESLastRoundInv(V state, const V round_key) {
state = detail::InvSubBytes(state);
state = detail::InvShiftRows(state);
state = Xor(state, round_key);
return state;
}
template <uint8_t kRcon, class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESKeyGenAssist(V v) {
alignas(16) static constexpr uint8_t kRconXorMask[16] = {
0, 0, 0, 0, kRcon, 0, 0, 0, 0, 0, 0, 0, kRcon, 0, 0, 0};
alignas(16) static constexpr uint8_t kRotWordShuffle[16] = {
4, 5, 6, 7, 5, 6, 7, 4, 12, 13, 14, 15, 13, 14, 15, 12};
const DFromV<decltype(v)> d;
const auto sub_word_result = detail::SubBytes(v);
const auto rot_word_result =
TableLookupBytes(sub_word_result, LoadDup128(d, kRotWordShuffle));
return Xor(rot_word_result, LoadDup128(d, kRconXorMask));
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CLMulLower(V a, V b) {
const DFromV<V> d;
static_assert(IsSame<TFromD<decltype(d)>, uint64_t>(), "V must be u64");
const auto k1 = Set(d, 0x1111111111111111ULL);
const auto k2 = Set(d, 0x2222222222222222ULL);
const auto k4 = Set(d, 0x4444444444444444ULL);
const auto k8 = Set(d, 0x8888888888888888ULL);
const auto a0 = And(a, k1);
const auto a1 = And(a, k2);
const auto a2 = And(a, k4);
const auto a3 = And(a, k8);
const auto b0 = And(b, k1);
const auto b1 = And(b, k2);
const auto b2 = And(b, k4);
const auto b3 = And(b, k8);
auto m0 = Xor(MulEven(a0, b0), MulEven(a1, b3));
auto m1 = Xor(MulEven(a0, b1), MulEven(a1, b0));
auto m2 = Xor(MulEven(a0, b2), MulEven(a1, b1));
auto m3 = Xor(MulEven(a0, b3), MulEven(a1, b2));
m0 = Xor(m0, Xor(MulEven(a2, b2), MulEven(a3, b1)));
m1 = Xor(m1, Xor(MulEven(a2, b3), MulEven(a3, b2)));
m2 = Xor(m2, Xor(MulEven(a2, b0), MulEven(a3, b3)));
m3 = Xor(m3, Xor(MulEven(a2, b1), MulEven(a3, b0)));
return Or(Or(And(m0, k1), And(m1, k2)), Or(And(m2, k4), And(m3, k8)));
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CLMulUpper(V a, V b) {
const DFromV<V> d;
static_assert(IsSame<TFromD<decltype(d)>, uint64_t>(), "V must be u64");
const auto k1 = Set(d, 0x1111111111111111ULL);
const auto k2 = Set(d, 0x2222222222222222ULL);
const auto k4 = Set(d, 0x4444444444444444ULL);
const auto k8 = Set(d, 0x8888888888888888ULL);
const auto a0 = And(a, k1);
const auto a1 = And(a, k2);
const auto a2 = And(a, k4);
const auto a3 = And(a, k8);
const auto b0 = And(b, k1);
const auto b1 = And(b, k2);
const auto b2 = And(b, k4);
const auto b3 = And(b, k8);
auto m0 = Xor(MulOdd(a0, b0), MulOdd(a1, b3));
auto m1 = Xor(MulOdd(a0, b1), MulOdd(a1, b0));
auto m2 = Xor(MulOdd(a0, b2), MulOdd(a1, b1));
auto m3 = Xor(MulOdd(a0, b3), MulOdd(a1, b2));
m0 = Xor(m0, Xor(MulOdd(a2, b2), MulOdd(a3, b1)));
m1 = Xor(m1, Xor(MulOdd(a2, b3), MulOdd(a3, b2)));
m2 = Xor(m2, Xor(MulOdd(a2, b0), MulOdd(a3, b3)));
m3 = Xor(m3, Xor(MulOdd(a2, b1), MulOdd(a3, b0)));
return Or(Or(And(m0, k1), And(m1, k2)), Or(And(m2, k4), And(m3, k8)));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr,
hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr, void* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
alignas(16) constexpr uint8_t kLookup[16] = {
0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
};
const auto lo = And(v, Set(d, uint8_t{0xF}));
const auto hi = ShiftRight<4>(v);
const auto lookup = LoadDup128(d, kLookup);
return Add(TableLookupBytes(lookup, hi), TableLookupBytes(lookup, lo));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
const V k33 = Set(d, uint8_t{0x33});
v = Sub(v, And(ShiftRight<1>(v), Set(d, uint8_t{0x55})));
v = Add(And(ShiftRight<2>(v), k33), And(v, k33));
return And(Add(v, ShiftRight<4>(v)), Set(d, uint8_t{0x0F}));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
const Repartition<uint8_t, decltype(d)> d8;
const auto vals = BitCast(d, PopulationCount(BitCast(d8, v)));
return Add(ShiftRight<8>(vals), And(vals, Set(d, uint16_t{0xFF})));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
Repartition<uint16_t, decltype(d)> d16;
auto vals = BitCast(d, PopulationCount(BitCast(d16, v)));
return Add(ShiftRight<16>(vals), And(vals, Set(d, uint32_t{0xFF})));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
Repartition<uint32_t, decltype(d)> d32;
auto vals = BitCast(d, PopulationCount(BitCast(d32, v)));
return Add(ShiftRight<32>(vals), And(vals, Set(d, 0xFFULL)));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (1)>* = nullptr,
hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16 / 2>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V operator*(const V a, const V b) {
const DFromV<decltype(a)> d;
const Rebind<MakeWide<TFromV<V>>, decltype(d)> dw;
const RebindToUnsigned<decltype(d)> du;
const RebindToUnsigned<decltype(dw)> dwu;
const VFromD<decltype(dw)> mul = PromoteTo(dw, a) * PromoteTo(dw, b);
return BitCast(d, TruncateTo(du, BitCast(dwu, mul)));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (1)>* = nullptr,
hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) > 16 / 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V operator*(const V a, const V b) {
const DFromV<decltype(a)> d;
const Half<decltype(d)> dh;
const Twice<RepartitionToWide<decltype(dh)>> dw;
const VFromD<decltype(dw)> a0 = PromoteTo(dw, LowerHalf(dh, a));
const VFromD<decltype(dw)> a1 = PromoteTo(dw, UpperHalf(dh, a));
const VFromD<decltype(dw)> b0 = PromoteTo(dw, LowerHalf(dh, b));
const VFromD<decltype(dw)> b1 = PromoteTo(dw, UpperHalf(dh, b));
const VFromD<decltype(dw)> m0 = a0 * b0;
const VFromD<decltype(dw)> m1 = a1 * b1;
return ConcatEven(d, BitCast(d, m1), BitCast(d, m0));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (8)>* = nullptr, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) == 8>* = nullptr,
hwy::EnableIf<!hwy::IsFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V operator*(V x, V y) {
const DFromV<V> d;
using T = TFromD<decltype(d)>;
using TU = MakeUnsigned<T>;
const TU xu = static_cast<TU>(GetLane(x));
const TU yu = static_cast<TU>(GetLane(y));
return Set(d, static_cast<T>(xu * yu));
}
template <class V, class D64 = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D64>, uint64_t>()>* = nullptr,
hwy::EnableIf<(D64::kPrivateLanes * sizeof(TFromD<D64>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V operator*(V x, V y) {
RepartitionToNarrow<D64> d32;
auto x32 = BitCast(d32, x);
auto y32 = BitCast(d32, y);
auto lolo = BitCast(d32, MulEven(x32, y32));
auto lohi = BitCast(d32, MulEven(x32, BitCast(d32, ShiftRight<32>(y))));
auto hilo = BitCast(d32, MulEven(BitCast(d32, ShiftRight<32>(x)), y32));
auto hi = BitCast(d32, ShiftLeft<32>(BitCast(D64{}, lohi + hilo)));
return BitCast(D64{}, lolo + hi);
}
template <class V, class DI64 = DFromV<V>, hwy::EnableIf<IsSame<TFromD<DI64>, int64_t>()>* = nullptr,
hwy::EnableIf<(DI64::kPrivateLanes * sizeof(TFromD<DI64>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V operator*(V x, V y) {
RebindToUnsigned<DI64> du64;
return BitCast(DI64{}, BitCast(du64, x) * BitCast(du64, y));
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V MulAdd(V mul, V x, V add) {
return Add(Mul(mul, x), add);
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V NegMulAdd(V mul, V x, V add) {
return Sub(add, Mul(mul, x));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ApproximateReciprocal(V v) {
const DFromV<decltype(v)> d;
return Div(Set(d, 1.0), v);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ApproximateReciprocalSqrt(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const auto half = Mul(v, Set(d, 0.5));
const auto guess = BitCast(d, Sub(Set(du, uint64_t{0x5FE6EB50C7B537A9u}),
ShiftRight<1>(BitCast(du, v))));
return Mul(guess, NegMulAdd(Mul(half, guess), guess, Set(d, 1.5)));
}
template <class V, class D, typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressBitsStore(V v, const uint8_t* __restrict__ bits, D d,
T* unaligned) {
alignas(16) T lanes[MaxLanes(d)];
Store(v, d, lanes);
const Simd<T, ((MaxLanes(d)) < (8) ? (MaxLanes(d)) : (8)), 0> d8;
T* __restrict__ pos = unaligned;
alignas(16) constexpr T table[2048] = {
0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
1, 0, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
2, 0, 1, 3, 4, 5, 6, 7, 0, 2, 1, 3, 4, 5, 6, 7,
1, 2, 0, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
3, 0, 1, 2, 4, 5, 6, 7, 0, 3, 1, 2, 4, 5, 6, 7,
1, 3, 0, 2, 4, 5, 6, 7, 0, 1, 3, 2, 4, 5, 6, 7,
2, 3, 0, 1, 4, 5, 6, 7, 0, 2, 3, 1, 4, 5, 6, 7,
1, 2, 3, 0, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
4, 0, 1, 2, 3, 5, 6, 7, 0, 4, 1, 2, 3, 5, 6, 7,
1, 4, 0, 2, 3, 5, 6, 7, 0, 1, 4, 2, 3, 5, 6, 7,
2, 4, 0, 1, 3, 5, 6, 7, 0, 2, 4, 1, 3, 5, 6, 7,
1, 2, 4, 0, 3, 5, 6, 7, 0, 1, 2, 4, 3, 5, 6, 7,
3, 4, 0, 1, 2, 5, 6, 7, 0, 3, 4, 1, 2, 5, 6, 7,
1, 3, 4, 0, 2, 5, 6, 7, 0, 1, 3, 4, 2, 5, 6, 7,
2, 3, 4, 0, 1, 5, 6, 7, 0, 2, 3, 4, 1, 5, 6, 7,
1, 2, 3, 4, 0, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
5, 0, 1, 2, 3, 4, 6, 7, 0, 5, 1, 2, 3, 4, 6, 7,
1, 5, 0, 2, 3, 4, 6, 7, 0, 1, 5, 2, 3, 4, 6, 7,
2, 5, 0, 1, 3, 4, 6, 7, 0, 2, 5, 1, 3, 4, 6, 7,
1, 2, 5, 0, 3, 4, 6, 7, 0, 1, 2, 5, 3, 4, 6, 7,
3, 5, 0, 1, 2, 4, 6, 7, 0, 3, 5, 1, 2, 4, 6, 7,
1, 3, 5, 0, 2, 4, 6, 7, 0, 1, 3, 5, 2, 4, 6, 7,
2, 3, 5, 0, 1, 4, 6, 7, 0, 2, 3, 5, 1, 4, 6, 7,
1, 2, 3, 5, 0, 4, 6, 7, 0, 1, 2, 3, 5, 4, 6, 7,
4, 5, 0, 1, 2, 3, 6, 7, 0, 4, 5, 1, 2, 3, 6, 7,
1, 4, 5, 0, 2, 3, 6, 7, 0, 1, 4, 5, 2, 3, 6, 7,
2, 4, 5, 0, 1, 3, 6, 7, 0, 2, 4, 5, 1, 3, 6, 7,
1, 2, 4, 5, 0, 3, 6, 7, 0, 1, 2, 4, 5, 3, 6, 7,
3, 4, 5, 0, 1, 2, 6, 7, 0, 3, 4, 5, 1, 2, 6, 7,
1, 3, 4, 5, 0, 2, 6, 7, 0, 1, 3, 4, 5, 2, 6, 7,
2, 3, 4, 5, 0, 1, 6, 7, 0, 2, 3, 4, 5, 1, 6, 7,
1, 2, 3, 4, 5, 0, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
6, 0, 1, 2, 3, 4, 5, 7, 0, 6, 1, 2, 3, 4, 5, 7,
1, 6, 0, 2, 3, 4, 5, 7, 0, 1, 6, 2, 3, 4, 5, 7,
2, 6, 0, 1, 3, 4, 5, 7, 0, 2, 6, 1, 3, 4, 5, 7,
1, 2, 6, 0, 3, 4, 5, 7, 0, 1, 2, 6, 3, 4, 5, 7,
3, 6, 0, 1, 2, 4, 5, 7, 0, 3, 6, 1, 2, 4, 5, 7,
1, 3, 6, 0, 2, 4, 5, 7, 0, 1, 3, 6, 2, 4, 5, 7,
2, 3, 6, 0, 1, 4, 5, 7, 0, 2, 3, 6, 1, 4, 5, 7,
1, 2, 3, 6, 0, 4, 5, 7, 0, 1, 2, 3, 6, 4, 5, 7,
4, 6, 0, 1, 2, 3, 5, 7, 0, 4, 6, 1, 2, 3, 5, 7,
1, 4, 6, 0, 2, 3, 5, 7, 0, 1, 4, 6, 2, 3, 5, 7,
2, 4, 6, 0, 1, 3, 5, 7, 0, 2, 4, 6, 1, 3, 5, 7,
1, 2, 4, 6, 0, 3, 5, 7, 0, 1, 2, 4, 6, 3, 5, 7,
3, 4, 6, 0, 1, 2, 5, 7, 0, 3, 4, 6, 1, 2, 5, 7,
1, 3, 4, 6, 0, 2, 5, 7, 0, 1, 3, 4, 6, 2, 5, 7,
2, 3, 4, 6, 0, 1, 5, 7, 0, 2, 3, 4, 6, 1, 5, 7,
1, 2, 3, 4, 6, 0, 5, 7, 0, 1, 2, 3, 4, 6, 5, 7,
5, 6, 0, 1, 2, 3, 4, 7, 0, 5, 6, 1, 2, 3, 4, 7,
1, 5, 6, 0, 2, 3, 4, 7, 0, 1, 5, 6, 2, 3, 4, 7,
2, 5, 6, 0, 1, 3, 4, 7, 0, 2, 5, 6, 1, 3, 4, 7,
1, 2, 5, 6, 0, 3, 4, 7, 0, 1, 2, 5, 6, 3, 4, 7,
3, 5, 6, 0, 1, 2, 4, 7, 0, 3, 5, 6, 1, 2, 4, 7,
1, 3, 5, 6, 0, 2, 4, 7, 0, 1, 3, 5, 6, 2, 4, 7,
2, 3, 5, 6, 0, 1, 4, 7, 0, 2, 3, 5, 6, 1, 4, 7,
1, 2, 3, 5, 6, 0, 4, 7, 0, 1, 2, 3, 5, 6, 4, 7,
4, 5, 6, 0, 1, 2, 3, 7, 0, 4, 5, 6, 1, 2, 3, 7,
1, 4, 5, 6, 0, 2, 3, 7, 0, 1, 4, 5, 6, 2, 3, 7,
2, 4, 5, 6, 0, 1, 3, 7, 0, 2, 4, 5, 6, 1, 3, 7,
1, 2, 4, 5, 6, 0, 3, 7, 0, 1, 2, 4, 5, 6, 3, 7,
3, 4, 5, 6, 0, 1, 2, 7, 0, 3, 4, 5, 6, 1, 2, 7,
1, 3, 4, 5, 6, 0, 2, 7, 0, 1, 3, 4, 5, 6, 2, 7,
2, 3, 4, 5, 6, 0, 1, 7, 0, 2, 3, 4, 5, 6, 1, 7,
1, 2, 3, 4, 5, 6, 0, 7, 0, 1, 2, 3, 4, 5, 6, 7,
7, 0, 1, 2, 3, 4, 5, 6, 0, 7, 1, 2, 3, 4, 5, 6,
1, 7, 0, 2, 3, 4, 5, 6, 0, 1, 7, 2, 3, 4, 5, 6,
2, 7, 0, 1, 3, 4, 5, 6, 0, 2, 7, 1, 3, 4, 5, 6,
1, 2, 7, 0, 3, 4, 5, 6, 0, 1, 2, 7, 3, 4, 5, 6,
3, 7, 0, 1, 2, 4, 5, 6, 0, 3, 7, 1, 2, 4, 5, 6,
1, 3, 7, 0, 2, 4, 5, 6, 0, 1, 3, 7, 2, 4, 5, 6,
2, 3, 7, 0, 1, 4, 5, 6, 0, 2, 3, 7, 1, 4, 5, 6,
1, 2, 3, 7, 0, 4, 5, 6, 0, 1, 2, 3, 7, 4, 5, 6,
4, 7, 0, 1, 2, 3, 5, 6, 0, 4, 7, 1, 2, 3, 5, 6,
1, 4, 7, 0, 2, 3, 5, 6, 0, 1, 4, 7, 2, 3, 5, 6,
2, 4, 7, 0, 1, 3, 5, 6, 0, 2, 4, 7, 1, 3, 5, 6,
1, 2, 4, 7, 0, 3, 5, 6, 0, 1, 2, 4, 7, 3, 5, 6,
3, 4, 7, 0, 1, 2, 5, 6, 0, 3, 4, 7, 1, 2, 5, 6,
1, 3, 4, 7, 0, 2, 5, 6, 0, 1, 3, 4, 7, 2, 5, 6,
2, 3, 4, 7, 0, 1, 5, 6, 0, 2, 3, 4, 7, 1, 5, 6,
1, 2, 3, 4, 7, 0, 5, 6, 0, 1, 2, 3, 4, 7, 5, 6,
5, 7, 0, 1, 2, 3, 4, 6, 0, 5, 7, 1, 2, 3, 4, 6,
1, 5, 7, 0, 2, 3, 4, 6, 0, 1, 5, 7, 2, 3, 4, 6,
2, 5, 7, 0, 1, 3, 4, 6, 0, 2, 5, 7, 1, 3, 4, 6,
1, 2, 5, 7, 0, 3, 4, 6, 0, 1, 2, 5, 7, 3, 4, 6,
3, 5, 7, 0, 1, 2, 4, 6, 0, 3, 5, 7, 1, 2, 4, 6,
1, 3, 5, 7, 0, 2, 4, 6, 0, 1, 3, 5, 7, 2, 4, 6,
2, 3, 5, 7, 0, 1, 4, 6, 0, 2, 3, 5, 7, 1, 4, 6,
1, 2, 3, 5, 7, 0, 4, 6, 0, 1, 2, 3, 5, 7, 4, 6,
4, 5, 7, 0, 1, 2, 3, 6, 0, 4, 5, 7, 1, 2, 3, 6,
1, 4, 5, 7, 0, 2, 3, 6, 0, 1, 4, 5, 7, 2, 3, 6,
2, 4, 5, 7, 0, 1, 3, 6, 0, 2, 4, 5, 7, 1, 3, 6,
1, 2, 4, 5, 7, 0, 3, 6, 0, 1, 2, 4, 5, 7, 3, 6,
3, 4, 5, 7, 0, 1, 2, 6, 0, 3, 4, 5, 7, 1, 2, 6,
1, 3, 4, 5, 7, 0, 2, 6, 0, 1, 3, 4, 5, 7, 2, 6,
2, 3, 4, 5, 7, 0, 1, 6, 0, 2, 3, 4, 5, 7, 1, 6,
1, 2, 3, 4, 5, 7, 0, 6, 0, 1, 2, 3, 4, 5, 7, 6,
6, 7, 0, 1, 2, 3, 4, 5, 0, 6, 7, 1, 2, 3, 4, 5,
1, 6, 7, 0, 2, 3, 4, 5, 0, 1, 6, 7, 2, 3, 4, 5,
2, 6, 7, 0, 1, 3, 4, 5, 0, 2, 6, 7, 1, 3, 4, 5,
1, 2, 6, 7, 0, 3, 4, 5, 0, 1, 2, 6, 7, 3, 4, 5,
3, 6, 7, 0, 1, 2, 4, 5, 0, 3, 6, 7, 1, 2, 4, 5,
1, 3, 6, 7, 0, 2, 4, 5, 0, 1, 3, 6, 7, 2, 4, 5,
2, 3, 6, 7, 0, 1, 4, 5, 0, 2, 3, 6, 7, 1, 4, 5,
1, 2, 3, 6, 7, 0, 4, 5, 0, 1, 2, 3, 6, 7, 4, 5,
4, 6, 7, 0, 1, 2, 3, 5, 0, 4, 6, 7, 1, 2, 3, 5,
1, 4, 6, 7, 0, 2, 3, 5, 0, 1, 4, 6, 7, 2, 3, 5,
2, 4, 6, 7, 0, 1, 3, 5, 0, 2, 4, 6, 7, 1, 3, 5,
1, 2, 4, 6, 7, 0, 3, 5, 0, 1, 2, 4, 6, 7, 3, 5,
3, 4, 6, 7, 0, 1, 2, 5, 0, 3, 4, 6, 7, 1, 2, 5,
1, 3, 4, 6, 7, 0, 2, 5, 0, 1, 3, 4, 6, 7, 2, 5,
2, 3, 4, 6, 7, 0, 1, 5, 0, 2, 3, 4, 6, 7, 1, 5,
1, 2, 3, 4, 6, 7, 0, 5, 0, 1, 2, 3, 4, 6, 7, 5,
5, 6, 7, 0, 1, 2, 3, 4, 0, 5, 6, 7, 1, 2, 3, 4,
1, 5, 6, 7, 0, 2, 3, 4, 0, 1, 5, 6, 7, 2, 3, 4,
2, 5, 6, 7, 0, 1, 3, 4, 0, 2, 5, 6, 7, 1, 3, 4,
1, 2, 5, 6, 7, 0, 3, 4, 0, 1, 2, 5, 6, 7, 3, 4,
3, 5, 6, 7, 0, 1, 2, 4, 0, 3, 5, 6, 7, 1, 2, 4,
1, 3, 5, 6, 7, 0, 2, 4, 0, 1, 3, 5, 6, 7, 2, 4,
2, 3, 5, 6, 7, 0, 1, 4, 0, 2, 3, 5, 6, 7, 1, 4,
1, 2, 3, 5, 6, 7, 0, 4, 0, 1, 2, 3, 5, 6, 7, 4,
4, 5, 6, 7, 0, 1, 2, 3, 0, 4, 5, 6, 7, 1, 2, 3,
1, 4, 5, 6, 7, 0, 2, 3, 0, 1, 4, 5, 6, 7, 2, 3,
2, 4, 5, 6, 7, 0, 1, 3, 0, 2, 4, 5, 6, 7, 1, 3,
1, 2, 4, 5, 6, 7, 0, 3, 0, 1, 2, 4, 5, 6, 7, 3,
3, 4, 5, 6, 7, 0, 1, 2, 0, 3, 4, 5, 6, 7, 1, 2,
1, 3, 4, 5, 6, 7, 0, 2, 0, 1, 3, 4, 5, 6, 7, 2,
2, 3, 4, 5, 6, 7, 0, 1, 0, 2, 3, 4, 5, 6, 7, 1,
1, 2, 3, 4, 5, 6, 7, 0, 0, 1, 2, 3, 4, 5, 6, 7};
for (size_t i = 0; i < Lanes(d); i += 8) {
const size_t bits8 = bits[i / 8];
const auto indices = Load(d8, table + bits8 * 8);
const auto compressed = TableLookupBytes(LoadU(d8, lanes + i), indices);
StoreU(compressed, d8, pos);
pos += PopCount(bits8);
}
return static_cast<size_t>(pos - unaligned);
}
template <class V, class M, class D, typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressStore(V v, M mask, D d, T* __restrict__ unaligned) {
uint8_t bits[((size_t{8}) > (MaxLanes(d) / 8) ? (size_t{8}) : (MaxLanes(d) / 8))];
(void)StoreMaskBits(d, mask, bits);
return CompressBitsStore(v, bits, d, unaligned);
}
template <class V, class M, class D, typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressBlendedStore(V v, M mask, D d,
T* __restrict__ unaligned) {
alignas(16) T buf[MaxLanes(d)];
const size_t bytes = CompressStore(v, mask, d, buf);
BlendedStore(Load(d, buf), FirstN(d, bytes), d, unaligned);
return bytes;
}
template <class V, class M, typename T = TFromV<V>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Compress(V v, const M mask) {
const DFromV<V> d;
alignas(16) T lanes[MaxLanes(d)];
(void)CompressStore(v, mask, d, lanes);
return Load(d, lanes);
}
template <class V, typename T = TFromV<V>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CompressBits(V v, const uint8_t* __restrict__ bits) {
const DFromV<V> d;
alignas(16) T lanes[MaxLanes(d)];
(void)CompressBitsStore(v, bits, d, lanes);
return Load(d, lanes);
}
template <class V, class M, typename T = TFromV<V>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CompressNot(V v, M mask) {
return Compress(v, Not(mask));
}
namespace detail {
template <size_t N>
inline __attribute__((always_inline)) Vec128<uint8_t, N> IndicesForExpandFromBits(uint64_t mask_bits) {
static_assert(N <= 8, "Should only be called for half-vectors");
const Simd<uint8_t, N, 0> du8;
do { } while (0);
alignas(16) static constexpr uint8_t table[2048] = {
128, 128, 128, 128, 128, 128, 128, 128,
0, 128, 128, 128, 128, 128, 128, 128,
128, 0, 128, 128, 128, 128, 128, 128,
0, 1, 128, 128, 128, 128, 128, 128,
128, 128, 0, 128, 128, 128, 128, 128,
0, 128, 1, 128, 128, 128, 128, 128,
128, 0, 1, 128, 128, 128, 128, 128,
0, 1, 2, 128, 128, 128, 128, 128,
128, 128, 128, 0, 128, 128, 128, 128,
0, 128, 128, 1, 128, 128, 128, 128,
128, 0, 128, 1, 128, 128, 128, 128,
0, 1, 128, 2, 128, 128, 128, 128,
128, 128, 0, 1, 128, 128, 128, 128,
0, 128, 1, 2, 128, 128, 128, 128,
128, 0, 1, 2, 128, 128, 128, 128,
0, 1, 2, 3, 128, 128, 128, 128,
128, 128, 128, 128, 0, 128, 128, 128,
0, 128, 128, 128, 1, 128, 128, 128,
128, 0, 128, 128, 1, 128, 128, 128,
0, 1, 128, 128, 2, 128, 128, 128,
128, 128, 0, 128, 1, 128, 128, 128,
0, 128, 1, 128, 2, 128, 128, 128,
128, 0, 1, 128, 2, 128, 128, 128,
0, 1, 2, 128, 3, 128, 128, 128,
128, 128, 128, 0, 1, 128, 128, 128,
0, 128, 128, 1, 2, 128, 128, 128,
128, 0, 128, 1, 2, 128, 128, 128,
0, 1, 128, 2, 3, 128, 128, 128,
128, 128, 0, 1, 2, 128, 128, 128,
0, 128, 1, 2, 3, 128, 128, 128,
128, 0, 1, 2, 3, 128, 128, 128,
0, 1, 2, 3, 4, 128, 128, 128,
128, 128, 128, 128, 128, 0, 128, 128,
0, 128, 128, 128, 128, 1, 128, 128,
128, 0, 128, 128, 128, 1, 128, 128,
0, 1, 128, 128, 128, 2, 128, 128,
128, 128, 0, 128, 128, 1, 128, 128,
0, 128, 1, 128, 128, 2, 128, 128,
128, 0, 1, 128, 128, 2, 128, 128,
0, 1, 2, 128, 128, 3, 128, 128,
128, 128, 128, 0, 128, 1, 128, 128,
0, 128, 128, 1, 128, 2, 128, 128,
128, 0, 128, 1, 128, 2, 128, 128,
0, 1, 128, 2, 128, 3, 128, 128,
128, 128, 0, 1, 128, 2, 128, 128,
0, 128, 1, 2, 128, 3, 128, 128,
128, 0, 1, 2, 128, 3, 128, 128,
0, 1, 2, 3, 128, 4, 128, 128,
128, 128, 128, 128, 0, 1, 128, 128,
0, 128, 128, 128, 1, 2, 128, 128,
128, 0, 128, 128, 1, 2, 128, 128,
0, 1, 128, 128, 2, 3, 128, 128,
128, 128, 0, 128, 1, 2, 128, 128,
0, 128, 1, 128, 2, 3, 128, 128,
128, 0, 1, 128, 2, 3, 128, 128,
0, 1, 2, 128, 3, 4, 128, 128,
128, 128, 128, 0, 1, 2, 128, 128,
0, 128, 128, 1, 2, 3, 128, 128,
128, 0, 128, 1, 2, 3, 128, 128,
0, 1, 128, 2, 3, 4, 128, 128,
128, 128, 0, 1, 2, 3, 128, 128,
0, 128, 1, 2, 3, 4, 128, 128,
128, 0, 1, 2, 3, 4, 128, 128,
0, 1, 2, 3, 4, 5, 128, 128,
128, 128, 128, 128, 128, 128, 0, 128,
0, 128, 128, 128, 128, 128, 1, 128,
128, 0, 128, 128, 128, 128, 1, 128,
0, 1, 128, 128, 128, 128, 2, 128,
128, 128, 0, 128, 128, 128, 1, 128,
0, 128, 1, 128, 128, 128, 2, 128,
128, 0, 1, 128, 128, 128, 2, 128,
0, 1, 2, 128, 128, 128, 3, 128,
128, 128, 128, 0, 128, 128, 1, 128,
0, 128, 128, 1, 128, 128, 2, 128,
128, 0, 128, 1, 128, 128, 2, 128,
0, 1, 128, 2, 128, 128, 3, 128,
128, 128, 0, 1, 128, 128, 2, 128,
0, 128, 1, 2, 128, 128, 3, 128,
128, 0, 1, 2, 128, 128, 3, 128,
0, 1, 2, 3, 128, 128, 4, 128,
128, 128, 128, 128, 0, 128, 1, 128,
0, 128, 128, 128, 1, 128, 2, 128,
128, 0, 128, 128, 1, 128, 2, 128,
0, 1, 128, 128, 2, 128, 3, 128,
128, 128, 0, 128, 1, 128, 2, 128,
0, 128, 1, 128, 2, 128, 3, 128,
128, 0, 1, 128, 2, 128, 3, 128,
0, 1, 2, 128, 3, 128, 4, 128,
128, 128, 128, 0, 1, 128, 2, 128,
0, 128, 128, 1, 2, 128, 3, 128,
128, 0, 128, 1, 2, 128, 3, 128,
0, 1, 128, 2, 3, 128, 4, 128,
128, 128, 0, 1, 2, 128, 3, 128,
0, 128, 1, 2, 3, 128, 4, 128,
128, 0, 1, 2, 3, 128, 4, 128,
0, 1, 2, 3, 4, 128, 5, 128,
128, 128, 128, 128, 128, 0, 1, 128,
0, 128, 128, 128, 128, 1, 2, 128,
128, 0, 128, 128, 128, 1, 2, 128,
0, 1, 128, 128, 128, 2, 3, 128,
128, 128, 0, 128, 128, 1, 2, 128,
0, 128, 1, 128, 128, 2, 3, 128,
128, 0, 1, 128, 128, 2, 3, 128,
0, 1, 2, 128, 128, 3, 4, 128,
128, 128, 128, 0, 128, 1, 2, 128,
0, 128, 128, 1, 128, 2, 3, 128,
128, 0, 128, 1, 128, 2, 3, 128,
0, 1, 128, 2, 128, 3, 4, 128,
128, 128, 0, 1, 128, 2, 3, 128,
0, 128, 1, 2, 128, 3, 4, 128,
128, 0, 1, 2, 128, 3, 4, 128,
0, 1, 2, 3, 128, 4, 5, 128,
128, 128, 128, 128, 0, 1, 2, 128,
0, 128, 128, 128, 1, 2, 3, 128,
128, 0, 128, 128, 1, 2, 3, 128,
0, 1, 128, 128, 2, 3, 4, 128,
128, 128, 0, 128, 1, 2, 3, 128,
0, 128, 1, 128, 2, 3, 4, 128,
128, 0, 1, 128, 2, 3, 4, 128,
0, 1, 2, 128, 3, 4, 5, 128,
128, 128, 128, 0, 1, 2, 3, 128,
0, 128, 128, 1, 2, 3, 4, 128,
128, 0, 128, 1, 2, 3, 4, 128,
0, 1, 128, 2, 3, 4, 5, 128,
128, 128, 0, 1, 2, 3, 4, 128,
0, 128, 1, 2, 3, 4, 5, 128,
128, 0, 1, 2, 3, 4, 5, 128,
0, 1, 2, 3, 4, 5, 6, 128,
128, 128, 128, 128, 128, 128, 128, 0,
0, 128, 128, 128, 128, 128, 128, 1,
128, 0, 128, 128, 128, 128, 128, 1,
0, 1, 128, 128, 128, 128, 128, 2,
128, 128, 0, 128, 128, 128, 128, 1,
0, 128, 1, 128, 128, 128, 128, 2,
128, 0, 1, 128, 128, 128, 128, 2,
0, 1, 2, 128, 128, 128, 128, 3,
128, 128, 128, 0, 128, 128, 128, 1,
0, 128, 128, 1, 128, 128, 128, 2,
128, 0, 128, 1, 128, 128, 128, 2,
0, 1, 128, 2, 128, 128, 128, 3,
128, 128, 0, 1, 128, 128, 128, 2,
0, 128, 1, 2, 128, 128, 128, 3,
128, 0, 1, 2, 128, 128, 128, 3,
0, 1, 2, 3, 128, 128, 128, 4,
128, 128, 128, 128, 0, 128, 128, 1,
0, 128, 128, 128, 1, 128, 128, 2,
128, 0, 128, 128, 1, 128, 128, 2,
0, 1, 128, 128, 2, 128, 128, 3,
128, 128, 0, 128, 1, 128, 128, 2,
0, 128, 1, 128, 2, 128, 128, 3,
128, 0, 1, 128, 2, 128, 128, 3,
0, 1, 2, 128, 3, 128, 128, 4,
128, 128, 128, 0, 1, 128, 128, 2,
0, 128, 128, 1, 2, 128, 128, 3,
128, 0, 128, 1, 2, 128, 128, 3,
0, 1, 128, 2, 3, 128, 128, 4,
128, 128, 0, 1, 2, 128, 128, 3,
0, 128, 1, 2, 3, 128, 128, 4,
128, 0, 1, 2, 3, 128, 128, 4,
0, 1, 2, 3, 4, 128, 128, 5,
128, 128, 128, 128, 128, 0, 128, 1,
0, 128, 128, 128, 128, 1, 128, 2,
128, 0, 128, 128, 128, 1, 128, 2,
0, 1, 128, 128, 128, 2, 128, 3,
128, 128, 0, 128, 128, 1, 128, 2,
0, 128, 1, 128, 128, 2, 128, 3,
128, 0, 1, 128, 128, 2, 128, 3,
0, 1, 2, 128, 128, 3, 128, 4,
128, 128, 128, 0, 128, 1, 128, 2,
0, 128, 128, 1, 128, 2, 128, 3,
128, 0, 128, 1, 128, 2, 128, 3,
0, 1, 128, 2, 128, 3, 128, 4,
128, 128, 0, 1, 128, 2, 128, 3,
0, 128, 1, 2, 128, 3, 128, 4,
128, 0, 1, 2, 128, 3, 128, 4,
0, 1, 2, 3, 128, 4, 128, 5,
128, 128, 128, 128, 0, 1, 128, 2,
0, 128, 128, 128, 1, 2, 128, 3,
128, 0, 128, 128, 1, 2, 128, 3,
0, 1, 128, 128, 2, 3, 128, 4,
128, 128, 0, 128, 1, 2, 128, 3,
0, 128, 1, 128, 2, 3, 128, 4,
128, 0, 1, 128, 2, 3, 128, 4,
0, 1, 2, 128, 3, 4, 128, 5,
128, 128, 128, 0, 1, 2, 128, 3,
0, 128, 128, 1, 2, 3, 128, 4,
128, 0, 128, 1, 2, 3, 128, 4,
0, 1, 128, 2, 3, 4, 128, 5,
128, 128, 0, 1, 2, 3, 128, 4,
0, 128, 1, 2, 3, 4, 128, 5,
128, 0, 1, 2, 3, 4, 128, 5,
0, 1, 2, 3, 4, 5, 128, 6,
128, 128, 128, 128, 128, 128, 0, 1,
0, 128, 128, 128, 128, 128, 1, 2,
128, 0, 128, 128, 128, 128, 1, 2,
0, 1, 128, 128, 128, 128, 2, 3,
128, 128, 0, 128, 128, 128, 1, 2,
0, 128, 1, 128, 128, 128, 2, 3,
128, 0, 1, 128, 128, 128, 2, 3,
0, 1, 2, 128, 128, 128, 3, 4,
128, 128, 128, 0, 128, 128, 1, 2,
0, 128, 128, 1, 128, 128, 2, 3,
128, 0, 128, 1, 128, 128, 2, 3,
0, 1, 128, 2, 128, 128, 3, 4,
128, 128, 0, 1, 128, 128, 2, 3,
0, 128, 1, 2, 128, 128, 3, 4,
128, 0, 1, 2, 128, 128, 3, 4,
0, 1, 2, 3, 128, 128, 4, 5,
128, 128, 128, 128, 0, 128, 1, 2,
0, 128, 128, 128, 1, 128, 2, 3,
128, 0, 128, 128, 1, 128, 2, 3,
0, 1, 128, 128, 2, 128, 3, 4,
128, 128, 0, 128, 1, 128, 2, 3,
0, 128, 1, 128, 2, 128, 3, 4,
128, 0, 1, 128, 2, 128, 3, 4,
0, 1, 2, 128, 3, 128, 4, 5,
128, 128, 128, 0, 1, 128, 2, 3,
0, 128, 128, 1, 2, 128, 3, 4,
128, 0, 128, 1, 2, 128, 3, 4,
0, 1, 128, 2, 3, 128, 4, 5,
128, 128, 0, 1, 2, 128, 3, 4,
0, 128, 1, 2, 3, 128, 4, 5,
128, 0, 1, 2, 3, 128, 4, 5,
0, 1, 2, 3, 4, 128, 5, 6,
128, 128, 128, 128, 128, 0, 1, 2,
0, 128, 128, 128, 128, 1, 2, 3,
128, 0, 128, 128, 128, 1, 2, 3,
0, 1, 128, 128, 128, 2, 3, 4,
128, 128, 0, 128, 128, 1, 2, 3,
0, 128, 1, 128, 128, 2, 3, 4,
128, 0, 1, 128, 128, 2, 3, 4,
0, 1, 2, 128, 128, 3, 4, 5,
128, 128, 128, 0, 128, 1, 2, 3,
0, 128, 128, 1, 128, 2, 3, 4,
128, 0, 128, 1, 128, 2, 3, 4,
0, 1, 128, 2, 128, 3, 4, 5,
128, 128, 0, 1, 128, 2, 3, 4,
0, 128, 1, 2, 128, 3, 4, 5,
128, 0, 1, 2, 128, 3, 4, 5,
0, 1, 2, 3, 128, 4, 5, 6,
128, 128, 128, 128, 0, 1, 2, 3,
0, 128, 128, 128, 1, 2, 3, 4,
128, 0, 128, 128, 1, 2, 3, 4,
0, 1, 128, 128, 2, 3, 4, 5,
128, 128, 0, 128, 1, 2, 3, 4,
0, 128, 1, 128, 2, 3, 4, 5,
128, 0, 1, 128, 2, 3, 4, 5,
0, 1, 2, 128, 3, 4, 5, 6,
128, 128, 128, 0, 1, 2, 3, 4,
0, 128, 128, 1, 2, 3, 4, 5,
128, 0, 128, 1, 2, 3, 4, 5,
0, 1, 128, 2, 3, 4, 5, 6,
128, 128, 0, 1, 2, 3, 4, 5,
0, 128, 1, 2, 3, 4, 5, 6,
128, 0, 1, 2, 3, 4, 5, 6,
0, 1, 2, 3, 4, 5, 6, 7};
return LoadU(du8, table + mask_bits * 8);
}
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr, hwy::EnableIf<N * sizeof(T) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Expand(Vec128<T, N> v, Mask128<T, N> mask) {
const DFromV<decltype(v)> d;
const uint64_t mask_bits = detail::BitsFromMask(mask);
const Vec128<uint8_t, N> indices =
detail::IndicesForExpandFromBits<N>(mask_bits);
return BitCast(d, TableLookupBytesOr0(v, indices));
}
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Expand(Vec128<T> v, Mask128<T> mask) {
const Full128<T> d;
const RebindToUnsigned<decltype(d)> du;
const Half<decltype(du)> duh;
const Vec128<uint8_t> vu = BitCast(du, v);
const uint64_t mask_bits = detail::BitsFromMask(mask);
const uint64_t maskL = mask_bits & 0xFF;
const uint64_t maskH = mask_bits >> 8;
alignas(16) static constexpr uint8_t iota[32] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128, 128, 128};
const VFromD<decltype(du)> shift = LoadU(du, iota + PopCount(maskL));
const VFromD<decltype(duh)> vL = LowerHalf(duh, vu);
const VFromD<decltype(duh)> vH =
LowerHalf(duh, TableLookupBytesOr0(vu, shift));
const VFromD<decltype(duh)> idxL = detail::IndicesForExpandFromBits<8>(maskL);
const VFromD<decltype(duh)> idxH = detail::IndicesForExpandFromBits<8>(maskH);
const VFromD<decltype(duh)> expandL = TableLookupBytesOr0(vL, idxL);
const VFromD<decltype(duh)> expandH = TableLookupBytesOr0(vH, idxH);
return BitCast(d, Combine(du, expandH, expandL));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Expand(Vec128<T, N> v, Mask128<T, N> mask) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const Rebind<uint8_t, decltype(d)> du8;
const uint64_t mask_bits = detail::BitsFromMask(mask);
alignas(16) static constexpr uint8_t table[2048] = {
128, 128, 128, 128, 128, 128, 128, 128,
0, 128, 128, 128, 128, 128, 128, 128,
128, 0, 128, 128, 128, 128, 128, 128,
0, 2, 128, 128, 128, 128, 128, 128,
128, 128, 0, 128, 128, 128, 128, 128,
0, 128, 2, 128, 128, 128, 128, 128,
128, 0, 2, 128, 128, 128, 128, 128,
0, 2, 4, 128, 128, 128, 128, 128,
128, 128, 128, 0, 128, 128, 128, 128,
0, 128, 128, 2, 128, 128, 128, 128,
128, 0, 128, 2, 128, 128, 128, 128,
0, 2, 128, 4, 128, 128, 128, 128,
128, 128, 0, 2, 128, 128, 128, 128,
0, 128, 2, 4, 128, 128, 128, 128,
128, 0, 2, 4, 128, 128, 128, 128,
0, 2, 4, 6, 128, 128, 128, 128,
128, 128, 128, 128, 0, 128, 128, 128,
0, 128, 128, 128, 2, 128, 128, 128,
128, 0, 128, 128, 2, 128, 128, 128,
0, 2, 128, 128, 4, 128, 128, 128,
128, 128, 0, 128, 2, 128, 128, 128,
0, 128, 2, 128, 4, 128, 128, 128,
128, 0, 2, 128, 4, 128, 128, 128,
0, 2, 4, 128, 6, 128, 128, 128,
128, 128, 128, 0, 2, 128, 128, 128,
0, 128, 128, 2, 4, 128, 128, 128,
128, 0, 128, 2, 4, 128, 128, 128,
0, 2, 128, 4, 6, 128, 128, 128,
128, 128, 0, 2, 4, 128, 128, 128,
0, 128, 2, 4, 6, 128, 128, 128,
128, 0, 2, 4, 6, 128, 128, 128,
0, 2, 4, 6, 8, 128, 128, 128,
128, 128, 128, 128, 128, 0, 128, 128,
0, 128, 128, 128, 128, 2, 128, 128,
128, 0, 128, 128, 128, 2, 128, 128,
0, 2, 128, 128, 128, 4, 128, 128,
128, 128, 0, 128, 128, 2, 128, 128,
0, 128, 2, 128, 128, 4, 128, 128,
128, 0, 2, 128, 128, 4, 128, 128,
0, 2, 4, 128, 128, 6, 128, 128,
128, 128, 128, 0, 128, 2, 128, 128,
0, 128, 128, 2, 128, 4, 128, 128,
128, 0, 128, 2, 128, 4, 128, 128,
0, 2, 128, 4, 128, 6, 128, 128,
128, 128, 0, 2, 128, 4, 128, 128,
0, 128, 2, 4, 128, 6, 128, 128,
128, 0, 2, 4, 128, 6, 128, 128,
0, 2, 4, 6, 128, 8, 128, 128,
128, 128, 128, 128, 0, 2, 128, 128,
0, 128, 128, 128, 2, 4, 128, 128,
128, 0, 128, 128, 2, 4, 128, 128,
0, 2, 128, 128, 4, 6, 128, 128,
128, 128, 0, 128, 2, 4, 128, 128,
0, 128, 2, 128, 4, 6, 128, 128,
128, 0, 2, 128, 4, 6, 128, 128,
0, 2, 4, 128, 6, 8, 128, 128,
128, 128, 128, 0, 2, 4, 128, 128,
0, 128, 128, 2, 4, 6, 128, 128,
128, 0, 128, 2, 4, 6, 128, 128,
0, 2, 128, 4, 6, 8, 128, 128,
128, 128, 0, 2, 4, 6, 128, 128,
0, 128, 2, 4, 6, 8, 128, 128,
128, 0, 2, 4, 6, 8, 128, 128,
0, 2, 4, 6, 8, 10, 128, 128,
128, 128, 128, 128, 128, 128, 0, 128,
0, 128, 128, 128, 128, 128, 2, 128,
128, 0, 128, 128, 128, 128, 2, 128,
0, 2, 128, 128, 128, 128, 4, 128,
128, 128, 0, 128, 128, 128, 2, 128,
0, 128, 2, 128, 128, 128, 4, 128,
128, 0, 2, 128, 128, 128, 4, 128,
0, 2, 4, 128, 128, 128, 6, 128,
128, 128, 128, 0, 128, 128, 2, 128,
0, 128, 128, 2, 128, 128, 4, 128,
128, 0, 128, 2, 128, 128, 4, 128,
0, 2, 128, 4, 128, 128, 6, 128,
128, 128, 0, 2, 128, 128, 4, 128,
0, 128, 2, 4, 128, 128, 6, 128,
128, 0, 2, 4, 128, 128, 6, 128,
0, 2, 4, 6, 128, 128, 8, 128,
128, 128, 128, 128, 0, 128, 2, 128,
0, 128, 128, 128, 2, 128, 4, 128,
128, 0, 128, 128, 2, 128, 4, 128,
0, 2, 128, 128, 4, 128, 6, 128,
128, 128, 0, 128, 2, 128, 4, 128,
0, 128, 2, 128, 4, 128, 6, 128,
128, 0, 2, 128, 4, 128, 6, 128,
0, 2, 4, 128, 6, 128, 8, 128,
128, 128, 128, 0, 2, 128, 4, 128,
0, 128, 128, 2, 4, 128, 6, 128,
128, 0, 128, 2, 4, 128, 6, 128,
0, 2, 128, 4, 6, 128, 8, 128,
128, 128, 0, 2, 4, 128, 6, 128,
0, 128, 2, 4, 6, 128, 8, 128,
128, 0, 2, 4, 6, 128, 8, 128,
0, 2, 4, 6, 8, 128, 10, 128,
128, 128, 128, 128, 128, 0, 2, 128,
0, 128, 128, 128, 128, 2, 4, 128,
128, 0, 128, 128, 128, 2, 4, 128,
0, 2, 128, 128, 128, 4, 6, 128,
128, 128, 0, 128, 128, 2, 4, 128,
0, 128, 2, 128, 128, 4, 6, 128,
128, 0, 2, 128, 128, 4, 6, 128,
0, 2, 4, 128, 128, 6, 8, 128,
128, 128, 128, 0, 128, 2, 4, 128,
0, 128, 128, 2, 128, 4, 6, 128,
128, 0, 128, 2, 128, 4, 6, 128,
0, 2, 128, 4, 128, 6, 8, 128,
128, 128, 0, 2, 128, 4, 6, 128,
0, 128, 2, 4, 128, 6, 8, 128,
128, 0, 2, 4, 128, 6, 8, 128,
0, 2, 4, 6, 128, 8, 10, 128,
128, 128, 128, 128, 0, 2, 4, 128,
0, 128, 128, 128, 2, 4, 6, 128,
128, 0, 128, 128, 2, 4, 6, 128,
0, 2, 128, 128, 4, 6, 8, 128,
128, 128, 0, 128, 2, 4, 6, 128,
0, 128, 2, 128, 4, 6, 8, 128,
128, 0, 2, 128, 4, 6, 8, 128,
0, 2, 4, 128, 6, 8, 10, 128,
128, 128, 128, 0, 2, 4, 6, 128,
0, 128, 128, 2, 4, 6, 8, 128,
128, 0, 128, 2, 4, 6, 8, 128,
0, 2, 128, 4, 6, 8, 10, 128,
128, 128, 0, 2, 4, 6, 8, 128,
0, 128, 2, 4, 6, 8, 10, 128,
128, 0, 2, 4, 6, 8, 10, 128,
0, 2, 4, 6, 8, 10, 12, 128,
128, 128, 128, 128, 128, 128, 128, 0,
0, 128, 128, 128, 128, 128, 128, 2,
128, 0, 128, 128, 128, 128, 128, 2,
0, 2, 128, 128, 128, 128, 128, 4,
128, 128, 0, 128, 128, 128, 128, 2,
0, 128, 2, 128, 128, 128, 128, 4,
128, 0, 2, 128, 128, 128, 128, 4,
0, 2, 4, 128, 128, 128, 128, 6,
128, 128, 128, 0, 128, 128, 128, 2,
0, 128, 128, 2, 128, 128, 128, 4,
128, 0, 128, 2, 128, 128, 128, 4,
0, 2, 128, 4, 128, 128, 128, 6,
128, 128, 0, 2, 128, 128, 128, 4,
0, 128, 2, 4, 128, 128, 128, 6,
128, 0, 2, 4, 128, 128, 128, 6,
0, 2, 4, 6, 128, 128, 128, 8,
128, 128, 128, 128, 0, 128, 128, 2,
0, 128, 128, 128, 2, 128, 128, 4,
128, 0, 128, 128, 2, 128, 128, 4,
0, 2, 128, 128, 4, 128, 128, 6,
128, 128, 0, 128, 2, 128, 128, 4,
0, 128, 2, 128, 4, 128, 128, 6,
128, 0, 2, 128, 4, 128, 128, 6,
0, 2, 4, 128, 6, 128, 128, 8,
128, 128, 128, 0, 2, 128, 128, 4,
0, 128, 128, 2, 4, 128, 128, 6,
128, 0, 128, 2, 4, 128, 128, 6,
0, 2, 128, 4, 6, 128, 128, 8,
128, 128, 0, 2, 4, 128, 128, 6,
0, 128, 2, 4, 6, 128, 128, 8,
128, 0, 2, 4, 6, 128, 128, 8,
0, 2, 4, 6, 8, 128, 128, 10,
128, 128, 128, 128, 128, 0, 128, 2,
0, 128, 128, 128, 128, 2, 128, 4,
128, 0, 128, 128, 128, 2, 128, 4,
0, 2, 128, 128, 128, 4, 128, 6,
128, 128, 0, 128, 128, 2, 128, 4,
0, 128, 2, 128, 128, 4, 128, 6,
128, 0, 2, 128, 128, 4, 128, 6,
0, 2, 4, 128, 128, 6, 128, 8,
128, 128, 128, 0, 128, 2, 128, 4,
0, 128, 128, 2, 128, 4, 128, 6,
128, 0, 128, 2, 128, 4, 128, 6,
0, 2, 128, 4, 128, 6, 128, 8,
128, 128, 0, 2, 128, 4, 128, 6,
0, 128, 2, 4, 128, 6, 128, 8,
128, 0, 2, 4, 128, 6, 128, 8,
0, 2, 4, 6, 128, 8, 128, 10,
128, 128, 128, 128, 0, 2, 128, 4,
0, 128, 128, 128, 2, 4, 128, 6,
128, 0, 128, 128, 2, 4, 128, 6,
0, 2, 128, 128, 4, 6, 128, 8,
128, 128, 0, 128, 2, 4, 128, 6,
0, 128, 2, 128, 4, 6, 128, 8,
128, 0, 2, 128, 4, 6, 128, 8,
0, 2, 4, 128, 6, 8, 128, 10,
128, 128, 128, 0, 2, 4, 128, 6,
0, 128, 128, 2, 4, 6, 128, 8,
128, 0, 128, 2, 4, 6, 128, 8,
0, 2, 128, 4, 6, 8, 128, 10,
128, 128, 0, 2, 4, 6, 128, 8,
0, 128, 2, 4, 6, 8, 128, 10,
128, 0, 2, 4, 6, 8, 128, 10,
0, 2, 4, 6, 8, 10, 128, 12,
128, 128, 128, 128, 128, 128, 0, 2,
0, 128, 128, 128, 128, 128, 2, 4,
128, 0, 128, 128, 128, 128, 2, 4,
0, 2, 128, 128, 128, 128, 4, 6,
128, 128, 0, 128, 128, 128, 2, 4,
0, 128, 2, 128, 128, 128, 4, 6,
128, 0, 2, 128, 128, 128, 4, 6,
0, 2, 4, 128, 128, 128, 6, 8,
128, 128, 128, 0, 128, 128, 2, 4,
0, 128, 128, 2, 128, 128, 4, 6,
128, 0, 128, 2, 128, 128, 4, 6,
0, 2, 128, 4, 128, 128, 6, 8,
128, 128, 0, 2, 128, 128, 4, 6,
0, 128, 2, 4, 128, 128, 6, 8,
128, 0, 2, 4, 128, 128, 6, 8,
0, 2, 4, 6, 128, 128, 8, 10,
128, 128, 128, 128, 0, 128, 2, 4,
0, 128, 128, 128, 2, 128, 4, 6,
128, 0, 128, 128, 2, 128, 4, 6,
0, 2, 128, 128, 4, 128, 6, 8,
128, 128, 0, 128, 2, 128, 4, 6,
0, 128, 2, 128, 4, 128, 6, 8,
128, 0, 2, 128, 4, 128, 6, 8,
0, 2, 4, 128, 6, 128, 8, 10,
128, 128, 128, 0, 2, 128, 4, 6,
0, 128, 128, 2, 4, 128, 6, 8,
128, 0, 128, 2, 4, 128, 6, 8,
0, 2, 128, 4, 6, 128, 8, 10,
128, 128, 0, 2, 4, 128, 6, 8,
0, 128, 2, 4, 6, 128, 8, 10,
128, 0, 2, 4, 6, 128, 8, 10,
0, 2, 4, 6, 8, 128, 10, 12,
128, 128, 128, 128, 128, 0, 2, 4,
0, 128, 128, 128, 128, 2, 4, 6,
128, 0, 128, 128, 128, 2, 4, 6,
0, 2, 128, 128, 128, 4, 6, 8,
128, 128, 0, 128, 128, 2, 4, 6,
0, 128, 2, 128, 128, 4, 6, 8,
128, 0, 2, 128, 128, 4, 6, 8,
0, 2, 4, 128, 128, 6, 8, 10,
128, 128, 128, 0, 128, 2, 4, 6,
0, 128, 128, 2, 128, 4, 6, 8,
128, 0, 128, 2, 128, 4, 6, 8,
0, 2, 128, 4, 128, 6, 8, 10,
128, 128, 0, 2, 128, 4, 6, 8,
0, 128, 2, 4, 128, 6, 8, 10,
128, 0, 2, 4, 128, 6, 8, 10,
0, 2, 4, 6, 128, 8, 10, 12,
128, 128, 128, 128, 0, 2, 4, 6,
0, 128, 128, 128, 2, 4, 6, 8,
128, 0, 128, 128, 2, 4, 6, 8,
0, 2, 128, 128, 4, 6, 8, 10,
128, 128, 0, 128, 2, 4, 6, 8,
0, 128, 2, 128, 4, 6, 8, 10,
128, 0, 2, 128, 4, 6, 8, 10,
0, 2, 4, 128, 6, 8, 10, 12,
128, 128, 128, 0, 2, 4, 6, 8,
0, 128, 128, 2, 4, 6, 8, 10,
128, 0, 128, 2, 4, 6, 8, 10,
0, 2, 128, 4, 6, 8, 10, 12,
128, 128, 0, 2, 4, 6, 8, 10,
0, 128, 2, 4, 6, 8, 10, 12,
128, 0, 2, 4, 6, 8, 10, 12,
0, 2, 4, 6, 8, 10, 12, 14};
const Twice<decltype(du8)> du8x2;
const Vec128<uint8_t, 2 * N> indices8 =
ZeroExtendVector(du8x2, Load(du8, table + mask_bits * 8));
const Vec128<uint16_t, N> indices16 =
BitCast(du, InterleaveLower(du8x2, indices8, indices8));
const Vec128<uint16_t, N> byte_indices = Add(indices16, Set(du, 0x0100));
return BitCast(d, TableLookupBytesOr0(v, byte_indices));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Expand(Vec128<T, N> v, Mask128<T, N> mask) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const uint64_t mask_bits = detail::BitsFromMask(mask);
alignas(16) static constexpr uint32_t packed_array[16] = {
0x0000ffff, 0x0000fff0, 0x0000ff0f, 0x0000ff10, 0x0000f0ff, 0x0000f1f0,
0x0000f10f, 0x0000f210, 0x00000fff, 0x00001ff0, 0x00001f0f, 0x00002f10,
0x000010ff, 0x000021f0, 0x0000210f, 0x00003210};
const Vec128<uint32_t, N> packed = Set(du, packed_array[mask_bits]);
alignas(16) static constexpr uint32_t shifts[4] = {0, 4, 8, 12};
Vec128<uint32_t, N> indices = packed >> Load(du, shifts);
indices = And(indices, Set(du, N - 1));
const Vec128<uint32_t, N> expand =
TableLookupLanes(BitCast(du, v), IndicesFromVec(du, indices));
return IfThenElseZero(mask, BitCast(d, expand));
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Expand(Vec128<T> v, Mask128<T> mask) {
return IfThenElseZero(mask, Compress(v, mask));
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> Expand(Vec128<T, 1> v, Mask128<T, 1> mask) {
return IfThenElseZero(mask, v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> LoadExpand(MFromD<D> mask, D d,
const TFromD<D>* __restrict__ unaligned) {
return Expand(LoadU(d, unaligned), mask);
}
template <class D>
using IndicesFromD = decltype(IndicesFromVec(D(), Zero(RebindToUnsigned<D>())));
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TwoTablesLookupLanes(D , VFromD<D> a, VFromD<D> b,
IndicesFromD<D> idx) {
return TwoTablesLookupLanes(a, b, idx);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse2(D d, VFromD<D> v) {
alignas(16) static constexpr TFromD<D> kShuffle[16] = {
1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14};
return TableLookupBytes(v, LoadDup128(d, kShuffle));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse4(D d, VFromD<D> v) {
alignas(16) static constexpr uint8_t kShuffle[16] = {
3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12};
const Repartition<uint8_t, decltype(d)> du8;
return TableLookupBytes(v, BitCast(d, LoadDup128(du8, kShuffle)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse8(D d, VFromD<D> v) {
alignas(16) static constexpr uint8_t kShuffle[16] = {
7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8};
const Repartition<uint8_t, decltype(d)> du8;
return TableLookupBytes(v, BitCast(d, LoadDup128(du8, kShuffle)));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseLaneBytes(V v) {
const DFromV<V> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, Reverse2(du8, BitCast(du8, v)));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseLaneBytes(V v) {
const DFromV<V> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, Reverse4(du8, BitCast(du8, v)));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseLaneBytes(V v) {
const DFromV<V> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, Reverse8(du8, BitCast(du8, v)));
}
namespace detail {
template <int kShiftAmt, int kShrResultMask, class V,
hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromD<DFromV<V> >) > 2 - 1)>* = nullptr>
inline __attribute__((always_inline)) V UI8ReverseBitsStep(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const Repartition<uint16_t, decltype(d)> d_shift;
const auto v_to_shift = BitCast(d_shift, v);
const auto shl_result = BitCast(d, ShiftLeft<kShiftAmt>(v_to_shift));
const auto shr_result = BitCast(d, ShiftRight<kShiftAmt>(v_to_shift));
const auto shr_result_mask =
BitCast(d, Set(du, static_cast<uint8_t>(kShrResultMask)));
return Or(And(shr_result, shr_result_mask),
AndNot(shr_result_mask, shl_result));
}
template <int kShiftAmt, int kShrResultMask, class V,
hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromD<DFromV<V> >) == 1>* = nullptr>
inline __attribute__((always_inline)) V UI8ReverseBitsStep(V v) {
return V{UI8ReverseBitsStep<kShiftAmt, kShrResultMask>(Vec128<uint8_t>{v.raw})
.raw};
}
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseBits(V v) {
auto result = detail::UI8ReverseBitsStep<1, 0x55>(v);
result = detail::UI8ReverseBitsStep<2, 0x33>(result);
result = detail::UI8ReverseBitsStep<4, 0x0F>(result);
return result;
}
template <class V, hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 2) | (1 << 4) | (1 << 8))) != 0>* = nullptr,
hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseBits(V v) {
const DFromV<decltype(v)> d;
const Repartition<uint8_t, decltype(d)> du8;
return ReverseLaneBytes(BitCast(d, ReverseBits(BitCast(du8, v))));
}
namespace detail {
template <class V>
inline __attribute__((always_inline)) V Per2LaneBlockShuffle(hwy::SizeTag<0> , V v) {
return DupEven(v);
}
template <class V>
inline __attribute__((always_inline)) V Per2LaneBlockShuffle(hwy::SizeTag<1> , V v) {
const DFromV<decltype(v)> d;
return Reverse2(d, v);
}
template <class V>
inline __attribute__((always_inline)) V Per2LaneBlockShuffle(hwy::SizeTag<2> , V v) {
return v;
}
template <class V>
inline __attribute__((always_inline)) V Per2LaneBlockShuffle(hwy::SizeTag<3> , V v) {
return DupOdd(v);
}
inline __attribute__((always_inline)) uint32_t U8x4Per4LaneBlkIndices(const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
return static_cast<uint32_t>((idx3 << 24) | (idx2 << 16) | (idx1 << 8) |
idx0);
}
template <class D>
inline __attribute__((always_inline)) Vec<D> TblLookupPer4LaneBlkU8IdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const Repartition<uint32_t, D> du32;
return ResizeBitCast(
d, Set(du32, U8x4Per4LaneBlkIndices(idx3, idx2, idx1, idx0)));
}
template <class V, hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2) | (1 << 4))) != 0>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlkShufDoTblLookup(V v, V idx) {
const DFromV<decltype(v)> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, TableLookupBytes(BitCast(du8, v), BitCast(du8, idx)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
inline __attribute__((always_inline)) Vec<D> TblLookupPer4LaneBlkShufIdx(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const Repartition<uint32_t, decltype(d)> du32;
const uint32_t idx3210 = U8x4Per4LaneBlkIndices(idx3, idx2, idx1, idx0);
const auto v_byte_idx = Per4LaneBlkShufDupSet4xU32(
du32, static_cast<uint32_t>(idx3210 + 0x0C0C0C0C),
static_cast<uint32_t>(idx3210 + 0x08080808),
static_cast<uint32_t>(idx3210 + 0x04040404),
static_cast<uint32_t>(idx3210));
return ResizeBitCast(d, v_byte_idx);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
inline __attribute__((always_inline)) Vec<D> TblLookupPer4LaneBlkShufIdx(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const Repartition<uint32_t, decltype(d)> du32;
const uint32_t idx10 = static_cast<uint32_t>((idx1 << 16) | idx0);
const uint32_t idx32 = static_cast<uint32_t>((idx3 << 16) | idx2);
constexpr uint32_t kLaneByteOffsets{0x01000100};
constexpr uint32_t kHiLaneByteOffsets{kLaneByteOffsets + 0x08080808u};
const auto v_byte_idx = Per4LaneBlkShufDupSet4xU32(
du32, static_cast<uint32_t>(idx32 * 0x0202u + kHiLaneByteOffsets),
static_cast<uint32_t>(idx10 * 0x0202u + kHiLaneByteOffsets),
static_cast<uint32_t>(idx32 * 0x0202u + kLaneByteOffsets),
static_cast<uint32_t>(idx10 * 0x0202u + kLaneByteOffsets));
return ResizeBitCast(d, v_byte_idx);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec<D> TblLookupPer4LaneBlkShufIdx(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const Repartition<uint32_t, decltype(d)> du32;
constexpr uint32_t kLaneByteOffsets{0x03020100};
const auto v_byte_idx = Per4LaneBlkShufDupSet4xU32(
du32, static_cast<uint32_t>(idx3 * 0x04040404u + kLaneByteOffsets),
static_cast<uint32_t>(idx2 * 0x04040404u + kLaneByteOffsets),
static_cast<uint32_t>(idx1 * 0x04040404u + kLaneByteOffsets),
static_cast<uint32_t>(idx0 * 0x04040404u + kLaneByteOffsets));
return ResizeBitCast(d, v_byte_idx);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> TblLookupPer4LaneBlkIdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
return TblLookupPer4LaneBlkU8IdxInBlk(d, idx3, idx2, idx1, idx0);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> TblLookupPer4LaneBlkIdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const uint16_t u16_idx0 = static_cast<uint16_t>(idx0);
const uint16_t u16_idx1 = static_cast<uint16_t>(idx1);
const uint16_t u16_idx2 = static_cast<uint16_t>(idx2);
const uint16_t u16_idx3 = static_cast<uint16_t>(idx3);
alignas(16)
const uint16_t indices[8] = {u16_idx0, u16_idx1, u16_idx2, u16_idx3,
u16_idx0, u16_idx1, u16_idx2, u16_idx3};
constexpr size_t kMinLanesToLoad = 8;
constexpr size_t kNumToLoad = ((D::kPrivateLanes) > (kMinLanesToLoad) ? (D::kPrivateLanes) : (kMinLanesToLoad));
const CappedTag<uint16_t, kNumToLoad> d_load;
return ResizeBitCast(d, LoadDup128(d_load, indices));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> TblLookupPer4LaneBlkIdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
return Per4LaneBlkShufDupSet4xU32(d, idx3, idx2, idx1, idx0);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> TblLookupPer4LaneBlkIdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const RebindToUnsigned<decltype(d)> du;
const Rebind<uint32_t, decltype(d)> du32;
return BitCast(d, PromoteTo(du, Per4LaneBlkShufDupSet4xU32(du32, idx3, idx2,
idx1, idx0)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
inline __attribute__((always_inline)) IndicesFromD<D> TblLookupPer4LaneBlkShufIdx(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
auto idx_in_blk = TblLookupPer4LaneBlkIdxInBlk(du, idx3, idx2, idx1, idx0);
constexpr size_t kN = D::kPrivateLanes;
if (kN < 4) {
idx_in_blk = And(idx_in_blk, Set(du, static_cast<TU>(kN - 1)));
}
const auto blk_offsets =
And(Iota(du, TU{0}), Set(du, static_cast<TU>(~TU{3})));
return IndicesFromVec(d, Add(idx_in_blk, blk_offsets));
}
template <class V, hwy::EnableIf<sizeof(TFromD<DFromV<V> >) == (8)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlkShufDoTblLookup(V v, IndicesFromD<DFromV<V>> idx) {
return TableLookupLanes(v, idx);
}
template <class V>
inline __attribute__((always_inline)) V TblLookupPer4LaneBlkShuf(V v, size_t idx3210) {
const DFromV<decltype(v)> d;
const uint32_t idx3 = static_cast<uint32_t>((idx3210 >> 6) & 3);
const uint32_t idx2 = static_cast<uint32_t>((idx3210 >> 4) & 3);
const uint32_t idx1 = static_cast<uint32_t>((idx3210 >> 2) & 3);
const uint32_t idx0 = static_cast<uint32_t>(idx3210 & 3);
const auto idx = TblLookupPer4LaneBlkShufIdx(d, idx3, idx2, idx1, idx0);
return Per4LaneBlkShufDoTblLookup(v, idx);
}
template <size_t kIdx3210, size_t kLaneSize, size_t kVectSize, class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> ,
hwy::SizeTag<kLaneSize> ,
hwy::SizeTag<kVectSize> ,
V v) {
return TblLookupPer4LaneBlkShuf(v, kIdx3210);
}
template <class V>
inline __attribute__((always_inline)) VFromD<RepartitionToWide<DFromV<V>>> Per4LaneBlockShufCastToWide(
hwy::FloatTag , hwy::SizeTag<4> , V v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> dw;
return BitCast(dw, v);
}
template <size_t kLaneSize, class V>
inline __attribute__((always_inline)) VFromD<RepartitionToWide<RebindToUnsigned<DFromV<V>>>>
Per4LaneBlockShufCastToWide(hwy::FloatTag ,
hwy::SizeTag<kLaneSize> , V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const RepartitionToWide<decltype(du)> dw;
return BitCast(dw, v);
}
template <size_t kLaneSize, class V>
inline __attribute__((always_inline)) VFromD<RepartitionToWide<DFromV<V>>> Per4LaneBlockShufCastToWide(
hwy::NonFloatTag ,
hwy::SizeTag<kLaneSize> , V v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> dw;
return BitCast(dw, v);
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x1B> , V v) {
const DFromV<decltype(v)> d;
return Reverse4(d, v);
}
template <class V,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2) | (1 ? (1 << 4) : 0))) != 0>* = nullptr
>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x44> , V v) {
const DFromV<decltype(v)> d;
const auto vw = Per4LaneBlockShufCastToWide(
hwy::IsFloatTag<TFromV<V>>(), hwy::SizeTag<sizeof(TFromV<V>)>(), v);
return BitCast(d, DupEven(vw));
}
template <class V,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2) | (1 ? (1 << 4) : 0))) != 0>* = nullptr
>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x4E> , V v) {
const DFromV<decltype(v)> d;
const auto vw = Per4LaneBlockShufCastToWide(
hwy::IsFloatTag<TFromV<V>>(), hwy::SizeTag<sizeof(TFromV<V>)>(), v);
const DFromV<decltype(vw)> dw;
return BitCast(d, Reverse2(dw, vw));
}
template <class V, hwy::EnableIf<(DFromV<V>::kPrivateLanes == 4)>* = nullptr,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2))) != 0>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x50> , V v) {
const DFromV<decltype(v)> d;
return InterleaveLower(d, v, v);
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (4)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x50> , V v) {
const DFromV<decltype(v)> d;
return InterleaveLower(d, v, v);
}
template <class V, hwy::EnableIf<(DFromV<V>::kPrivateLanes == 4)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x88> , V v) {
const DFromV<decltype(v)> d;
return ConcatEven(d, v, v);
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xA0> , V v) {
return DupEven(v);
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xB1> , V v) {
const DFromV<decltype(v)> d;
return Reverse2(d, v);
}
template <class V, hwy::EnableIf<(DFromV<V>::kPrivateLanes == 4)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xDD> , V v) {
const DFromV<decltype(v)> d;
return ConcatOdd(d, v, v);
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xE4> , V v) {
return v;
}
template <class V,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2) | (1 ? (1 << 4) : 0))) != 0>* = nullptr
>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xEE> , V v) {
const DFromV<decltype(v)> d;
const auto vw = Per4LaneBlockShufCastToWide(
hwy::IsFloatTag<TFromV<V>>(), hwy::SizeTag<sizeof(TFromV<V>)>(), v);
return BitCast(d, DupOdd(vw));
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xF5> , V v) {
return DupOdd(v);
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (4)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xFA> , V v) {
const DFromV<decltype(v)> d;
return InterleaveUpper(d, v, v);
}
template <size_t kIdx3210, class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> idx_3210_tag, V v) {
const DFromV<decltype(v)> d;
return Per4LaneBlockShuffle(idx_3210_tag, hwy::SizeTag<sizeof(TFromV<V>)>(),
hwy::SizeTag<d.MaxBytes()>(), v);
}
}
template <size_t kIdx3, size_t kIdx2, size_t kIdx1, size_t kIdx0, class V,
hwy::EnableIf<(DFromV<V>::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Per4LaneBlockShuffle(V v) {
static_assert(kIdx0 <= 3, "kIdx0 <= 3 must be true");
static_assert(kIdx1 <= 3, "kIdx1 <= 3 must be true");
static_assert(kIdx2 <= 3, "kIdx2 <= 3 must be true");
static_assert(kIdx3 <= 3, "kIdx3 <= 3 must be true");
return v;
}
template <size_t kIdx3, size_t kIdx2, size_t kIdx1, size_t kIdx0, class V,
hwy::EnableIf<(DFromV<V>::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Per4LaneBlockShuffle(V v) {
static_assert(kIdx0 <= 3, "kIdx0 <= 3 must be true");
static_assert(kIdx1 <= 3, "kIdx1 <= 3 must be true");
static_assert(kIdx2 <= 3, "kIdx2 <= 3 must be true");
static_assert(kIdx3 <= 3, "kIdx3 <= 3 must be true");
constexpr bool isReverse2 = (kIdx0 == 1 || kIdx1 == 0) && (kIdx0 != kIdx1);
constexpr size_t kPer2BlkIdx0 = (kIdx0 <= 1) ? kIdx0 : (isReverse2 ? 1 : 0);
constexpr size_t kPer2BlkIdx1 = (kIdx1 <= 1) ? kIdx1 : (isReverse2 ? 0 : 1);
constexpr size_t kIdx10 = (kPer2BlkIdx1 << 1) | kPer2BlkIdx0;
static_assert(kIdx10 <= 3, "kIdx10 <= 3 must be true");
return detail::Per2LaneBlockShuffle(hwy::SizeTag<kIdx10>(), v);
}
template <size_t kIdx3, size_t kIdx2, size_t kIdx1, size_t kIdx0, class V,
hwy::EnableIf<(DFromV<V>::kPrivateLanes > 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Per4LaneBlockShuffle(V v) {
static_assert(kIdx0 <= 3, "kIdx0 <= 3 must be true");
static_assert(kIdx1 <= 3, "kIdx1 <= 3 must be true");
static_assert(kIdx2 <= 3, "kIdx2 <= 3 must be true");
static_assert(kIdx3 <= 3, "kIdx3 <= 3 must be true");
constexpr size_t kIdx3210 =
(kIdx3 << 6) | (kIdx2 << 4) | (kIdx1 << 2) | kIdx0;
return detail::Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210>(), v);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t Blocks(D d) {
return (d.MaxBytes() <= 16) ? 1 : ((Lanes(d) * sizeof(TFromD<D>) + 15) / 16);
}
template <int kBlockIdx, class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V InsertBlock(V , V blk_to_insert) {
static_assert(kBlockIdx == 0, "Invalid block index");
return blk_to_insert;
}
template <int kBlockIdx, class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ExtractBlock(V v) {
static_assert(kBlockIdx == 0, "Invalid block index");
return v;
}
template <int kBlockIdx, class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V BroadcastBlock(V v) {
static_assert(kBlockIdx == 0, "Invalid block index");
return v;
}
template<int kLane, class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V BroadcastLane(V v) {
return Broadcast<kLane>(v);
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Slide1Up(D d, VFromD<D> ) {
return Zero(d);
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Slide1Down(D d, VFromD<D> ) {
return Zero(d);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Slide1Up(D d, VFromD<D> v) {
return ShiftLeftLanes<1>(d, v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Slide1Down(D d, VFromD<D> v) {
return ShiftRightLanes<1>(d, v);
}
template <int kBlocks, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpBlocks(D , VFromD<D> v) {
static_assert(kBlocks == 0, "kBlocks == 0 must be true");
return v;
}
template <int kBlocks, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownBlocks(D , VFromD<D> v) {
static_assert(kBlocks == 0, "kBlocks == 0 must be true");
return v;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Add(V a, V b) {
return a + b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Sub(V a, V b) {
return a - b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Mul(V a, V b) {
return a * b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Div(V a, V b) {
return a / b;
}
template <class V>
V Shl(V a, V b) {
return a << b;
}
template <class V>
V Shr(V a, V b) {
return a >> b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Eq(V a, V b) -> decltype(a == b) {
return a == b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Ne(V a, V b) -> decltype(a == b) {
return a != b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Lt(V a, V b) -> decltype(a == b) {
return a < b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Gt(V a, V b) -> decltype(a == b) {
return a > b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Ge(V a, V b) -> decltype(a == b) {
return a >= b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Le(V a, V b) -> decltype(a == b) {
return a <= b;
}
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC push_options
#pragma GCC target "sse2,ssse3"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSSE3 {
template <class D, class V>
inline __attribute__((always_inline)) V Acos(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAcos(const D d, VecArg<V> x) {
return Acos(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Acosh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAcosh(const D d, VecArg<V> x) {
return Acosh(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Asin(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAsin(const D d, VecArg<V> x) {
return Asin(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Asinh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAsinh(const D d, VecArg<V> x) {
return Asinh(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Atan(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAtan(const D d, VecArg<V> x) {
return Atan(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Atanh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAtanh(const D d, VecArg<V> x) {
return Atanh(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Cos(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallCos(const D d, VecArg<V> x) {
return Cos(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Exp(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallExp(const D d, VecArg<V> x) {
return Exp(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Expm1(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallExpm1(const D d, VecArg<V> x) {
return Expm1(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallLog(const D d, VecArg<V> x) {
return Log(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log10(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallLog10(const D d, VecArg<V> x) {
return Log10(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log1p(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallLog1p(const D d, VecArg<V> x) {
return Log1p(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log2(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallLog2(const D d, VecArg<V> x) {
return Log2(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Sin(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallSin(const D d, VecArg<V> x) {
return Sin(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Sinh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallSinh(const D d, VecArg<V> x) {
return Sinh(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Tanh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallTanh(const D d, VecArg<V> x) {
return Tanh(d, x);
}
namespace impl {
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1) {
return MulAdd(c1, x, c0);
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2) {
T x2 = Mul(x, x);
return MulAdd(x2, c2, MulAdd(c1, x, c0));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3) {
T x2 = Mul(x, x);
return MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
return MulAdd(x4, c4, MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
return MulAdd(x4, MulAdd(c5, x, c4),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
return MulAdd(x4, MulAdd(x2, c6, MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
return MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8, c8,
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8, MulAdd(c9, x, c8),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8, MulAdd(x2, c10, MulAdd(c9, x, c8)),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8, MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8)),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(
x8, MulAdd(x4, c12, MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8,
MulAdd(x4, MulAdd(c13, x, c12),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8,
MulAdd(x4, MulAdd(x2, c14, MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14, T c15) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8,
MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14, T c15, T c16) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
T x16 = Mul(x8, x8);
return MulAdd(
x16, c16,
MulAdd(x8,
MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14, T c15, T c16, T c17) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
T x16 = Mul(x8, x8);
return MulAdd(
x16, MulAdd(c17, x, c16),
MulAdd(x8,
MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14, T c15, T c16, T c17,
T c18) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
T x16 = Mul(x8, x8);
return MulAdd(
x16, MulAdd(x2, c18, MulAdd(c17, x, c16)),
MulAdd(x8,
MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))));
}
template <class FloatOrDouble>
struct AsinImpl {};
template <class FloatOrDouble>
struct AtanImpl {};
template <class FloatOrDouble>
struct CosSinImpl {};
template <class FloatOrDouble>
struct ExpImpl {};
template <class FloatOrDouble>
struct LogImpl {};
template <>
struct AsinImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) V AsinPoly(D d, V x2, V ) {
const auto k0 = Set(d, +0.1666677296f);
const auto k1 = Set(d, +0.07495029271f);
const auto k2 = Set(d, +0.04547423869f);
const auto k3 = Set(d, +0.02424046025f);
const auto k4 = Set(d, +0.04197454825f);
return Estrin(x2, k0, k1, k2, k3, k4);
}
};
template <>
struct AsinImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) V AsinPoly(D d, V x2, V ) {
const auto k0 = Set(d, +0.1666666666666497543);
const auto k1 = Set(d, +0.07500000000378581611);
const auto k2 = Set(d, +0.04464285681377102438);
const auto k3 = Set(d, +0.03038195928038132237);
const auto k4 = Set(d, +0.02237176181932048341);
const auto k5 = Set(d, +0.01735956991223614604);
const auto k6 = Set(d, +0.01388715184501609218);
const auto k7 = Set(d, +0.01215360525577377331);
const auto k8 = Set(d, +0.006606077476277170610);
const auto k9 = Set(d, +0.01929045477267910674);
const auto k10 = Set(d, -0.01581918243329996643);
const auto k11 = Set(d, +0.03161587650653934628);
return Estrin(x2, k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11);
}
};
template <>
struct AtanImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) V AtanPoly(D d, V x) {
const auto k0 = Set(d, -0.333331018686294555664062f);
const auto k1 = Set(d, +0.199926957488059997558594f);
const auto k2 = Set(d, -0.142027363181114196777344f);
const auto k3 = Set(d, +0.106347933411598205566406f);
const auto k4 = Set(d, -0.0748900920152664184570312f);
const auto k5 = Set(d, +0.0425049886107444763183594f);
const auto k6 = Set(d, -0.0159569028764963150024414f);
const auto k7 = Set(d, +0.00282363896258175373077393f);
const auto y = Mul(x, x);
return MulAdd(Estrin(y, k0, k1, k2, k3, k4, k5, k6, k7), Mul(y, x), x);
}
};
template <>
struct AtanImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) V AtanPoly(D d, V x) {
const auto k0 = Set(d, -0.333333333333311110369124);
const auto k1 = Set(d, +0.199999999996591265594148);
const auto k2 = Set(d, -0.14285714266771329383765);
const auto k3 = Set(d, +0.111111105648261418443745);
const auto k4 = Set(d, -0.090908995008245008229153);
const auto k5 = Set(d, +0.0769219538311769618355029);
const auto k6 = Set(d, -0.0666573579361080525984562);
const auto k7 = Set(d, +0.0587666392926673580854313);
const auto k8 = Set(d, -0.0523674852303482457616113);
const auto k9 = Set(d, +0.0466667150077840625632675);
const auto k10 = Set(d, -0.0407629191276836500001934);
const auto k11 = Set(d, +0.0337852580001353069993897);
const auto k12 = Set(d, -0.0254517624932312641616861);
const auto k13 = Set(d, +0.016599329773529201970117);
const auto k14 = Set(d, -0.00889896195887655491740809);
const auto k15 = Set(d, +0.00370026744188713119232403);
const auto k16 = Set(d, -0.00110611831486672482563471);
const auto k17 = Set(d, +0.000209850076645816976906797);
const auto k18 = Set(d, -1.88796008463073496563746e-5);
const auto y = Mul(x, x);
return MulAdd(Estrin(y, k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11,
k12, k13, k14, k15, k16, k17, k18),
Mul(y, x), x);
}
};
template <>
struct CosSinImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> ToInt32(D , V x) {
return ConvertTo(Rebind<int32_t, D>(), x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Poly(D d, V x) {
const auto k0 = Set(d, -1.66666597127914428710938e-1f);
const auto k1 = Set(d, +8.33307858556509017944336e-3f);
const auto k2 = Set(d, -1.981069071916863322258e-4f);
const auto k3 = Set(d, +2.6083159809786593541503e-6f);
const auto y = Mul(x, x);
return MulAdd(Estrin(y, k0, k1, k2, k3), Mul(y, x), x);
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V CosReduce(D d, V x, VI32 q) {
const V kHalfPiPart0f = Set(d, -0.5f * 3.140625f);
const V kHalfPiPart1f = Set(d, -0.5f * 0.0009670257568359375f);
const V kHalfPiPart2f = Set(d, -0.5f * 6.2771141529083251953e-7f);
const V kHalfPiPart3f = Set(d, -0.5f * 1.2154201256553420762e-10f);
const V qf = ConvertTo(d, q);
x = MulAdd(qf, kHalfPiPart0f, x);
x = MulAdd(qf, kHalfPiPart1f, x);
x = MulAdd(qf, kHalfPiPart2f, x);
x = MulAdd(qf, kHalfPiPart3f, x);
return x;
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V SinReduce(D d, V x, VI32 q) {
const V kPiPart0f = Set(d, -3.140625f);
const V kPiPart1f = Set(d, -0.0009670257568359375f);
const V kPiPart2f = Set(d, -6.2771141529083251953e-7f);
const V kPiPart3f = Set(d, -1.2154201256553420762e-10f);
const V qf = ConvertTo(d, q);
x = MulAdd(qf, kPiPart0f, x);
x = MulAdd(qf, kPiPart1f, x);
x = MulAdd(qf, kPiPart2f, x);
x = MulAdd(qf, kPiPart3f, x);
return x;
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<Rebind<float, D>> CosSignFromQuadrant(D d, VI32 q) {
const VI32 kTwo = Set(Rebind<int32_t, D>(), 2);
return BitCast(d, ShiftLeft<30>(AndNot(q, kTwo)));
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<Rebind<float, D>> SinSignFromQuadrant(D d, VI32 q) {
const VI32 kOne = Set(Rebind<int32_t, D>(), 1);
return BitCast(d, ShiftLeft<31>(And(q, kOne)));
}
};
template <>
struct CosSinImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> ToInt32(D , V x) {
return DemoteTo(Rebind<int32_t, D>(), x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Poly(D d, V x) {
const auto k0 = Set(d, -0.166666666666666657414808);
const auto k1 = Set(d, +0.00833333333333332974823815);
const auto k2 = Set(d, -0.000198412698412696162806809);
const auto k3 = Set(d, +2.75573192239198747630416e-6);
const auto k4 = Set(d, -2.50521083763502045810755e-8);
const auto k5 = Set(d, +1.60590430605664501629054e-10);
const auto k6 = Set(d, -7.64712219118158833288484e-13);
const auto k7 = Set(d, +2.81009972710863200091251e-15);
const auto k8 = Set(d, -7.97255955009037868891952e-18);
const auto y = Mul(x, x);
return MulAdd(Estrin(y, k0, k1, k2, k3, k4, k5, k6, k7, k8), Mul(y, x), x);
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V CosReduce(D d, V x, VI32 q) {
const V kHalfPiPart0d = Set(d, -0.5 * 3.1415926218032836914);
const V kHalfPiPart1d = Set(d, -0.5 * 3.1786509424591713469e-8);
const V kHalfPiPart2d = Set(d, -0.5 * 1.2246467864107188502e-16);
const V kHalfPiPart3d = Set(d, -0.5 * 1.2736634327021899816e-24);
const V qf = PromoteTo(d, q);
x = MulAdd(qf, kHalfPiPart0d, x);
x = MulAdd(qf, kHalfPiPart1d, x);
x = MulAdd(qf, kHalfPiPart2d, x);
x = MulAdd(qf, kHalfPiPart3d, x);
return x;
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V SinReduce(D d, V x, VI32 q) {
const V kPiPart0d = Set(d, -3.1415926218032836914);
const V kPiPart1d = Set(d, -3.1786509424591713469e-8);
const V kPiPart2d = Set(d, -1.2246467864107188502e-16);
const V kPiPart3d = Set(d, -1.2736634327021899816e-24);
const V qf = PromoteTo(d, q);
x = MulAdd(qf, kPiPart0d, x);
x = MulAdd(qf, kPiPart1d, x);
x = MulAdd(qf, kPiPart2d, x);
x = MulAdd(qf, kPiPart3d, x);
return x;
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<Rebind<double, D>> CosSignFromQuadrant(D d, VI32 q) {
const VI32 kTwo = Set(Rebind<int32_t, D>(), 2);
return BitCast(
d, ShiftLeft<62>(PromoteTo(Rebind<int64_t, D>(), AndNot(q, kTwo))));
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<Rebind<double, D>> SinSignFromQuadrant(D d, VI32 q) {
const VI32 kOne = Set(Rebind<int32_t, D>(), 1);
return BitCast(
d, ShiftLeft<63>(PromoteTo(Rebind<int64_t, D>(), And(q, kOne))));
}
};
template <>
struct ExpImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> ToInt32(D , V x) {
return ConvertTo(Rebind<int32_t, D>(), x);
}
template <class D, class V>
inline __attribute__((always_inline)) V ExpPoly(D d, V x) {
const auto k0 = Set(d, +0.5f);
const auto k1 = Set(d, +0.166666671633720397949219f);
const auto k2 = Set(d, +0.0416664853692054748535156f);
const auto k3 = Set(d, +0.00833336077630519866943359f);
const auto k4 = Set(d, +0.00139304355252534151077271f);
const auto k5 = Set(d, +0.000198527617612853646278381f);
return MulAdd(Estrin(x, k0, k1, k2, k3, k4, k5), Mul(x, x), x);
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<D> Pow2I(D d, VI32 x) {
const Rebind<int32_t, D> di32;
const VI32 kOffset = Set(di32, 0x7F);
return BitCast(d, ShiftLeft<23>(Add(x, kOffset)));
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V LoadExpShortRange(D d, V x, VI32 e) {
const VI32 y = ShiftRight<1>(e);
return Mul(Mul(x, Pow2I(d, y)), Pow2I(d, Sub(e, y)));
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V ExpReduce(D d, V x, VI32 q) {
const V kLn2Part0f = Set(d, -0.693145751953125f);
const V kLn2Part1f = Set(d, -1.428606765330187045e-6f);
const V qf = ConvertTo(d, q);
x = MulAdd(qf, kLn2Part0f, x);
x = MulAdd(qf, kLn2Part1f, x);
return x;
}
};
template <>
struct LogImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> Log2p1NoSubnormal(D , V x) {
const Rebind<int32_t, D> di32;
const Rebind<uint32_t, D> du32;
const auto kBias = Set(di32, 0x7F);
return Sub(BitCast(di32, ShiftRight<23>(BitCast(du32, x))), kBias);
}
template <class D, class V>
inline __attribute__((always_inline)) V LogPoly(D d, V x) {
const V k0 = Set(d, 0.66666662693f);
const V k1 = Set(d, 0.40000972152f);
const V k2 = Set(d, 0.28498786688f);
const V k3 = Set(d, 0.24279078841f);
const V x2 = Mul(x, x);
const V x4 = Mul(x2, x2);
return MulAdd(MulAdd(k2, x4, k0), x2, Mul(MulAdd(k3, x4, k1), x4));
}
};
template <>
struct ExpImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> ToInt32(D , V x) {
return DemoteTo(Rebind<int32_t, D>(), x);
}
template <class D, class V>
inline __attribute__((always_inline)) V ExpPoly(D d, V x) {
const auto k0 = Set(d, +0.5);
const auto k1 = Set(d, +0.166666666666666851703837);
const auto k2 = Set(d, +0.0416666666666665047591422);
const auto k3 = Set(d, +0.00833333333331652721664984);
const auto k4 = Set(d, +0.00138888888889774492207962);
const auto k5 = Set(d, +0.000198412698960509205564975);
const auto k6 = Set(d, +2.4801587159235472998791e-5);
const auto k7 = Set(d, +2.75572362911928827629423e-6);
const auto k8 = Set(d, +2.75573911234900471893338e-7);
const auto k9 = Set(d, +2.51112930892876518610661e-8);
const auto k10 = Set(d, +2.08860621107283687536341e-9);
return MulAdd(Estrin(x, k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10),
Mul(x, x), x);
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<D> Pow2I(D d, VI32 x) {
const Rebind<int32_t, D> di32;
const Rebind<int64_t, D> di64;
const VI32 kOffset = Set(di32, 0x3FF);
return BitCast(d, ShiftLeft<52>(PromoteTo(di64, Add(x, kOffset))));
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V LoadExpShortRange(D d, V x, VI32 e) {
const VI32 y = ShiftRight<1>(e);
return Mul(Mul(x, Pow2I(d, y)), Pow2I(d, Sub(e, y)));
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V ExpReduce(D d, V x, VI32 q) {
const V kLn2Part0d = Set(d, -0.6931471805596629565116018);
const V kLn2Part1d = Set(d, -0.28235290563031577122588448175e-12);
const V qf = PromoteTo(d, q);
x = MulAdd(qf, kLn2Part0d, x);
x = MulAdd(qf, kLn2Part1d, x);
return x;
}
};
template <>
struct LogImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int64_t, D>> Log2p1NoSubnormal(D , V x) {
const Rebind<int64_t, D> di64;
const Rebind<uint64_t, D> du64;
return Sub(BitCast(di64, ShiftRight<52>(BitCast(du64, x))),
Set(di64, 0x3FF));
}
template <class D, class V>
inline __attribute__((always_inline)) V LogPoly(D d, V x) {
const V k0 = Set(d, 0.6666666666666735130);
const V k1 = Set(d, 0.3999999999940941908);
const V k2 = Set(d, 0.2857142874366239149);
const V k3 = Set(d, 0.2222219843214978396);
const V k4 = Set(d, 0.1818357216161805012);
const V k5 = Set(d, 0.1531383769920937332);
const V k6 = Set(d, 0.1479819860511658591);
const V x2 = Mul(x, x);
const V x4 = Mul(x2, x2);
return MulAdd(MulAdd(MulAdd(MulAdd(k6, x4, k4), x4, k2), x4, k0), x2,
(Mul(MulAdd(MulAdd(k5, x4, k3), x4, k1), x4)));
}
};
template <class D, class V, bool kAllowSubnormals = true>
inline __attribute__((always_inline)) V Log(const D d, V x) {
using T = TFromD<D>;
impl::LogImpl<T> impl;
constexpr bool kIsF32 = (sizeof(T) == 4);
const V kLn2Hi = Set(d, kIsF32 ? static_cast<T>(0.69313812256f)
: static_cast<T>(0.693147180369123816490));
const V kLn2Lo = Set(d, kIsF32 ? static_cast<T>(9.0580006145e-6f)
: static_cast<T>(1.90821492927058770002e-10));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kMinNormal = Set(d, kIsF32 ? static_cast<T>(1.175494351e-38f)
: static_cast<T>(2.2250738585072014e-308));
const V kScale = Set(d, kIsF32 ? static_cast<T>(3.355443200e+7f)
: static_cast<T>(1.8014398509481984e+16));
using TI = MakeSigned<T>;
const Rebind<TI, D> di;
using VI = decltype(Zero(di));
const VI kLowerBits = Set(di, kIsF32 ? static_cast<TI>(0x00000000L)
: static_cast<TI>(0xFFFFFFFFLL));
const VI kMagic = Set(di, kIsF32 ? static_cast<TI>(0x3F3504F3L)
: static_cast<TI>(0x3FE6A09E00000000LL));
const VI kExpMask = Set(di, kIsF32 ? static_cast<TI>(0x3F800000L)
: static_cast<TI>(0x3FF0000000000000LL));
const VI kExpScale =
Set(di, kIsF32 ? static_cast<TI>(-25) : static_cast<TI>(-54));
const VI kManMask = Set(di, kIsF32 ? static_cast<TI>(0x7FFFFFL)
: static_cast<TI>(0xFFFFF00000000LL));
VI exp_bits;
V exp;
if (kAllowSubnormals == true) {
const auto is_denormal = Lt(x, kMinNormal);
x = IfThenElse(is_denormal, Mul(x, kScale), x);
exp_bits = Add(BitCast(di, x), Sub(kExpMask, kMagic));
const VI exp_scale =
BitCast(di, IfThenElseZero(is_denormal, BitCast(d, kExpScale)));
exp = ConvertTo(
d, Add(exp_scale, impl.Log2p1NoSubnormal(d, BitCast(d, exp_bits))));
} else {
exp_bits = Add(BitCast(di, x), Sub(kExpMask, kMagic));
exp = ConvertTo(d, impl.Log2p1NoSubnormal(d, BitCast(d, exp_bits)));
}
const V y = Or(And(x, BitCast(d, kLowerBits)),
BitCast(d, Add(And(exp_bits, kManMask), kMagic)));
const V ym1 = Sub(y, kOne);
const V z = Div(ym1, Add(y, kOne));
return MulSub(
exp, kLn2Hi,
Sub(MulSub(z, Sub(ym1, impl.LogPoly(d, z)), Mul(exp, kLn2Lo)), ym1));
}
}
template <class D, class V>
inline __attribute__((always_inline)) V Acos(const D d, V x) {
using T = TFromD<D>;
const V kZero = Zero(d);
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kPi = Set(d, static_cast<T>(+3.14159265358979323846264));
const V kPiOverTwo = Set(d, static_cast<T>(+1.57079632679489661923132169));
const V sign_x = And(SignBit(d), x);
const V abs_x = Xor(x, sign_x);
const auto mask = Lt(abs_x, kHalf);
const V yy =
IfThenElse(mask, Mul(abs_x, abs_x), NegMulAdd(abs_x, kHalf, kHalf));
const V y = IfThenElse(mask, abs_x, Sqrt(yy));
impl::AsinImpl<T> impl;
const V t = Mul(impl.AsinPoly(d, yy, y), Mul(y, yy));
const V t_plus_y = Add(t, y);
const V z =
IfThenElse(mask, Sub(kPiOverTwo, Add(Xor(y, sign_x), Xor(t, sign_x))),
Add(t_plus_y, t_plus_y));
return IfThenElse(Or(mask, Ge(x, kZero)), z, Sub(kPi, z));
}
template <class D, class V>
inline __attribute__((always_inline)) V Acosh(const D d, V x) {
using T = TFromD<D>;
const V kLarge = Set(d, static_cast<T>(268435456.0));
const V kLog2 = Set(d, static_cast<T>(0.693147180559945286227));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kTwo = Set(d, static_cast<T>(+2.0));
const auto is_x_large = Gt(x, kLarge);
const auto is_x_gt_2 = Gt(x, kTwo);
const V x_minus_1 = Sub(x, kOne);
const V y0 = MulSub(kTwo, x, Div(kOne, Add(Sqrt(MulSub(x, x, kOne)), x)));
const V y1 =
Add(Sqrt(MulAdd(x_minus_1, kTwo, Mul(x_minus_1, x_minus_1))), x_minus_1);
const V y2 =
IfThenElse(is_x_gt_2, IfThenElse(is_x_large, x, y0), Add(y1, kOne));
const V z = impl::Log<D, V, false>(d, y2);
const auto is_pole = Eq(y2, kOne);
const auto divisor = Sub(IfThenZeroElse(is_pole, y2), kOne);
return Add(IfThenElse(is_x_gt_2, z,
IfThenElse(is_pole, y1, Div(Mul(z, y1), divisor))),
IfThenElseZero(is_x_large, kLog2));
}
template <class D, class V>
inline __attribute__((always_inline)) V Asin(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kTwo = Set(d, static_cast<T>(+2.0));
const V kPiOverTwo = Set(d, static_cast<T>(+1.57079632679489661923132169));
const V sign_x = And(SignBit(d), x);
const V abs_x = Xor(x, sign_x);
const auto mask = Lt(abs_x, kHalf);
const V yy =
IfThenElse(mask, Mul(abs_x, abs_x), NegMulAdd(abs_x, kHalf, kHalf));
const V y = IfThenElse(mask, abs_x, Sqrt(yy));
impl::AsinImpl<T> impl;
const V z0 = MulAdd(impl.AsinPoly(d, yy, y), Mul(yy, y), y);
const V z1 = NegMulAdd(z0, kTwo, kPiOverTwo);
return Or(IfThenElse(mask, z0, z1), sign_x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Asinh(const D d, V x) {
using T = TFromD<D>;
const V kSmall = Set(d, static_cast<T>(1.0 / 268435456.0));
const V kLarge = Set(d, static_cast<T>(268435456.0));
const V kLog2 = Set(d, static_cast<T>(0.693147180559945286227));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kTwo = Set(d, static_cast<T>(+2.0));
const V sign_x = And(SignBit(d), x);
const V abs_x = Xor(x, sign_x);
const auto is_x_large = Gt(abs_x, kLarge);
const auto is_x_lt_2 = Lt(abs_x, kTwo);
const V x2 = Mul(x, x);
const V sqrt_x2_plus_1 = Sqrt(Add(x2, kOne));
const V y0 = MulAdd(abs_x, kTwo, Div(kOne, Add(sqrt_x2_plus_1, abs_x)));
const V y1 = Add(Div(x2, Add(sqrt_x2_plus_1, kOne)), abs_x);
const V y2 =
IfThenElse(is_x_lt_2, Add(y1, kOne), IfThenElse(is_x_large, abs_x, y0));
const V z = impl::Log<D, V, false>(d, y2);
const auto is_pole = Eq(y2, kOne);
const auto divisor = Sub(IfThenZeroElse(is_pole, y2), kOne);
const auto large = IfThenElse(is_pole, y1, Div(Mul(z, y1), divisor));
const V y = IfThenElse(Lt(abs_x, kSmall), x, large);
return Or(Add(IfThenElse(is_x_lt_2, y, z), IfThenElseZero(is_x_large, kLog2)),
sign_x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Atan(const D d, V x) {
using T = TFromD<D>;
const V kOne = Set(d, static_cast<T>(+1.0));
const V kPiOverTwo = Set(d, static_cast<T>(+1.57079632679489661923132169));
const V sign = And(SignBit(d), x);
const V abs_x = Xor(x, sign);
const auto mask = Gt(abs_x, kOne);
impl::AtanImpl<T> impl;
const auto divisor = IfThenElse(mask, abs_x, kOne);
const V y = impl.AtanPoly(d, IfThenElse(mask, Div(kOne, divisor), abs_x));
return Or(IfThenElse(mask, Sub(kPiOverTwo, y), y), sign);
}
template <class D, class V>
inline __attribute__((always_inline)) V Atanh(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kOne = Set(d, static_cast<T>(+1.0));
const V sign = And(SignBit(d), x);
const V abs_x = Xor(x, sign);
return Mul(Log1p(d, Div(Add(abs_x, abs_x), Sub(kOne, abs_x))),
Xor(kHalf, sign));
}
template <class D, class V>
inline __attribute__((always_inline)) V Cos(const D d, V x) {
using T = TFromD<D>;
impl::CosSinImpl<T> impl;
const V kOneOverPi = Set(d, static_cast<T>(0.31830988618379067153));
const Rebind<int32_t, D> di32;
using VI32 = decltype(Zero(di32));
const VI32 kOne = Set(di32, 1);
const V y = Abs(x);
const VI32 q = Add(ShiftLeft<1>(impl.ToInt32(d, Mul(y, kOneOverPi))), kOne);
return impl.Poly(
d, Xor(impl.CosReduce(d, y, q), impl.CosSignFromQuadrant(d, q)));
}
template <class D, class V>
inline __attribute__((always_inline)) V Exp(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kLowerBound =
Set(d, static_cast<T>((sizeof(T) == 4 ? -104.0 : -1000.0)));
const V kNegZero = Set(d, static_cast<T>(-0.0));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kOneOverLog2 = Set(d, static_cast<T>(+1.442695040888963407359924681));
impl::ExpImpl<T> impl;
const auto q =
impl.ToInt32(d, MulAdd(x, kOneOverLog2, Or(kHalf, And(x, kNegZero))));
const V y = impl.LoadExpShortRange(
d, Add(impl.ExpPoly(d, impl.ExpReduce(d, x, q)), kOne), q);
return IfThenElseZero(Ge(x, kLowerBound), y);
}
template <class D, class V>
inline __attribute__((always_inline)) V Expm1(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kLowerBound =
Set(d, static_cast<T>((sizeof(T) == 4 ? -104.0 : -1000.0)));
const V kLn2Over2 = Set(d, static_cast<T>(+0.346573590279972654708616));
const V kNegOne = Set(d, static_cast<T>(-1.0));
const V kNegZero = Set(d, static_cast<T>(-0.0));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kOneOverLog2 = Set(d, static_cast<T>(+1.442695040888963407359924681));
impl::ExpImpl<T> impl;
const auto q =
impl.ToInt32(d, MulAdd(x, kOneOverLog2, Or(kHalf, And(x, kNegZero))));
const V y = impl.ExpPoly(d, impl.ExpReduce(d, x, q));
const V z = IfThenElse(Lt(Abs(x), kLn2Over2), y,
Sub(impl.LoadExpShortRange(d, Add(y, kOne), q), kOne));
return IfThenElse(Lt(x, kLowerBound), kNegOne, z);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log(const D d, V x) {
return impl::Log<D, V, true>(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log10(const D d, V x) {
using T = TFromD<D>;
return Mul(Log(d, x), Set(d, static_cast<T>(0.4342944819032518276511)));
}
template <class D, class V>
inline __attribute__((always_inline)) V Log1p(const D d, V x) {
using T = TFromD<D>;
const V kOne = Set(d, static_cast<T>(+1.0));
const V y = Add(x, kOne);
const auto is_pole = Eq(y, kOne);
const auto divisor = Sub(IfThenZeroElse(is_pole, y), kOne);
const auto non_pole =
Mul(impl::Log<D, V, false>(d, y), Div(x, divisor));
return IfThenElse(is_pole, x, non_pole);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log2(const D d, V x) {
using T = TFromD<D>;
return Mul(Log(d, x), Set(d, static_cast<T>(1.44269504088896340735992)));
}
template <class D, class V>
inline __attribute__((always_inline)) V Sin(const D d, V x) {
using T = TFromD<D>;
impl::CosSinImpl<T> impl;
const V kOneOverPi = Set(d, static_cast<T>(0.31830988618379067153));
const V kHalf = Set(d, static_cast<T>(0.5));
const Rebind<int32_t, D> di32;
using VI32 = decltype(Zero(di32));
const V abs_x = Abs(x);
const V sign_x = Xor(abs_x, x);
const VI32 q = impl.ToInt32(d, MulAdd(abs_x, kOneOverPi, kHalf));
return impl.Poly(d, Xor(impl.SinReduce(d, abs_x, q),
Xor(impl.SinSignFromQuadrant(d, q), sign_x)));
}
template <class D, class V>
inline __attribute__((always_inline)) V Sinh(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kTwo = Set(d, static_cast<T>(+2.0));
const V sign = And(SignBit(d), x);
const V abs_x = Xor(x, sign);
const V y = Expm1(d, abs_x);
const V z = Mul(Div(Add(y, kTwo), Add(y, kOne)), Mul(y, kHalf));
return Xor(z, sign);
}
template <class D, class V>
inline __attribute__((always_inline)) V Tanh(const D d, V x) {
using T = TFromD<D>;
const V kLimit = Set(d, static_cast<T>(18.714973875));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kTwo = Set(d, static_cast<T>(+2.0));
const V sign = And(SignBit(d), x);
const V abs_x = Xor(x, sign);
const V y = Expm1(d, Mul(abs_x, kTwo));
const V z = IfThenElse(Gt(abs_x, kLimit), kOne, Div(y, Add(y, kTwo)));
return Xor(z, sign);
}
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC push_options
#pragma GCC target "sse2,ssse3"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSSE3 {
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Print(const D d, const char* caption, V v, size_t lane_u = 0,
size_t max_lanes = 7) {
const size_t N = Lanes(d);
using T = TFromD<D>;
auto lanes = AllocateAligned<T>(N);
Store(v, d, lanes.get());
const auto info = hwy::detail::MakeTypeInfo<T>();
hwy::detail::PrintArray(info, caption, lanes.get(), N, lane_u, max_lanes);
}
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC push_options
#pragma GCC target "sse2,ssse3"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSSE3 {
template <class D, typename T = TFromD<D>>
inline __attribute__((always_inline)) void AssertVecEqual(D d, const T* expected, Vec<D> actual,
const char* filename, const int line) {
const size_t N = Lanes(d);
auto actual_lanes = AllocateAligned<T>(N);
Store(actual, d, actual_lanes.get());
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName((1LL << 12));
hwy::detail::AssertArrayEqual(info, expected, actual_lanes.get(), N,
target_name, filename, line);
}
template <class D, typename T = TFromD<D>>
inline __attribute__((always_inline)) void AssertVecEqual(D d, Vec<D> expected, Vec<D> actual,
const char* filename, int line) {
const size_t N = Lanes(d);
auto expected_lanes = AllocateAligned<T>(N);
auto actual_lanes = AllocateAligned<T>(N);
Store(expected, d, expected_lanes.get());
Store(actual, d, actual_lanes.get());
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName((1LL << 12));
hwy::detail::AssertArrayEqual(info, expected_lanes.get(), actual_lanes.get(),
N, target_name, filename, line);
}
template <class D>
__attribute__((noinline)) void AssertMaskEqual(D d, VecArg<Mask<D>> a, VecArg<Mask<D>> b,
const char* filename, int line) {
const Vec<D> va = VecFromMask(d, a);
const Vec<D> vb = VecFromMask(d, b);
AssertVecEqual(d, va, vb, filename, line);
const char* target_name = hwy::TargetName((1LL << 12));
AssertEqual(CountTrue(d, a), CountTrue(d, b), target_name, filename, line);
AssertEqual(AllTrue(d, a), AllTrue(d, b), target_name, filename, line);
AssertEqual(AllFalse(d, a), AllFalse(d, b), target_name, filename, line);
const size_t N = Lanes(d);
const Repartition<uint8_t, D> d8;
const size_t N8 = Lanes(d8);
auto bits_a = AllocateAligned<uint8_t>(((size_t{8}) > (N8) ? (size_t{8}) : (N8)));
auto bits_b = AllocateAligned<uint8_t>(size_t{((8) > (N8) ? (8) : (N8))});
memset(bits_a.get(), 0, N8);
memset(bits_b.get(), 0, N8);
const size_t num_bytes_a = StoreMaskBits(d, a, bits_a.get());
const size_t num_bytes_b = StoreMaskBits(d, b, bits_b.get());
AssertEqual(num_bytes_a, num_bytes_b, target_name, filename, line);
size_t i = 0;
for (; i < N / 8; ++i) {
if (bits_a[i] != bits_b[i]) {
fprintf(
stderr
, "Mismatch in byte %d: %d != %d\n", static_cast<int>(i),
bits_a[i], bits_b[i]);
Print(d8, "expect", Load(d8, bits_a.get()), 0, N8);
Print(d8, "actual", Load(d8, bits_b.get()), 0, N8);
hwy::Abort(filename, line, "Masks not equal");
}
}
const size_t remainder = N % 8;
if (remainder != 0) {
const int mask = (1 << remainder) - 1;
const int valid_a = bits_a[i] & mask;
const int valid_b = bits_b[i] & mask;
if (valid_a != valid_b) {
fprintf(
stderr
, "Mismatch in last byte %d: %d != %d\n",
static_cast<int>(i), valid_a, valid_b);
Print(d8, "expect", Load(d8, bits_a.get()), 0, N8);
Print(d8, "actual", Load(d8, bits_b.get()), 0, N8);
hwy::Abort(filename, line, "Masks not equal");
}
}
}
template <class D>
inline __attribute__((always_inline)) Mask<D> MaskTrue(const D d) {
return FirstN(d, Lanes(d));
}
template <class D>
inline __attribute__((always_inline)) Mask<D> MaskFalse(const D d) {
const RebindToSigned<D> di;
const Vec<decltype(di)> zero = Zero(di);
return RebindMask(d, Lt(zero, zero));
}
namespace detail {
template <typename T, size_t kMul, size_t kMinArg, class Test, int kPow2 = 0>
struct ForeachCappedR {
static void Do(size_t min_lanes, size_t max_lanes) {
const CappedTag<T, kMul * kMinArg, kPow2> d;
const size_t lanes = Lanes(d);
if (lanes < min_lanes) return;
if (lanes <= max_lanes) {
Test()(T(), d);
}
ForeachCappedR<T, kMul / 2, kMinArg, Test, kPow2>::Do(min_lanes, max_lanes);
}
};
template <typename T, size_t kMinArg, class Test, int kPow2>
struct ForeachCappedR<T, 0, kMinArg, Test, kPow2> {
static void Do(size_t, size_t) {}
};
}
template <class Test>
class ForMaxPow2 {
mutable bool called_ = false;
public:
~ForMaxPow2() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 291, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
detail::ForeachCappedR<T, (16 / sizeof(T)), 1, Test>::Do(
1, Lanes(ScalableTag<T>()));
}
};
template <class Test, int kPow2 = 1>
class ForExtendableVectors {
mutable bool called_ = false;
public:
~ForExtendableVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 324, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMaxCapped = (16 / sizeof(T));
const size_t max_lanes = Lanes(ScalableTag<T>()) >> kPow2;
(void)kMaxCapped;
(void)max_lanes;
constexpr size_t kMul = kMaxCapped >> kPow2;
constexpr size_t kMinArg = size_t{1} << kPow2;
detail::ForeachCappedR<T, kMul, kMinArg, Test, -kPow2>::Do(1, max_lanes);
}
};
template <class Test, int kPow2 = 1>
class ForShrinkableVectors {
mutable bool called_ = false;
public:
~ForShrinkableVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 358, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMinLanes = size_t{1} << kPow2;
constexpr size_t kMaxCapped = (16 / sizeof(T));
constexpr size_t max_lanes = kMaxCapped;
(void)kMinLanes;
(void)max_lanes;
(void)max_lanes;
detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kMinLanes, Test>::Do(
kMinLanes, max_lanes);
}
};
template <size_t kMinBits, class Test>
class ForGEVectors {
mutable bool called_ = false;
public:
~ForGEVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 394, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMaxCapped = (16 / sizeof(T));
constexpr size_t kMinLanes = kMinBits / 8 / sizeof(T);
constexpr size_t max_lanes = kMaxCapped;
(void)max_lanes;
detail::ForeachCappedR<T, (16 / sizeof(T)) / kMinLanes, kMinLanes, Test>::Do(
kMinLanes, max_lanes);
}
};
template <class Test>
using ForGE128Vectors = ForGEVectors<128, Test>;
template <class Test, int kPow2 = 1>
class ForPromoteVectors {
mutable bool called_ = false;
public:
~ForPromoteVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 436, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kFactor = size_t{1} << kPow2;
static_assert(kFactor >= 2 && kFactor * sizeof(T) <= sizeof(uint64_t), "");
constexpr size_t kMaxCapped = (16 / sizeof(T));
const size_t max_lanes = Lanes(ScalableTag<T>()) >> kPow2;
(void)kMaxCapped;
(void)max_lanes;
using DLargestFrom = CappedTag<T, (kMaxCapped >> kPow2) * kFactor, -kPow2>;
static_assert(DLargestFrom::kPrivateLanes <= (kMaxCapped >> kPow2),
"HWY_MAX_LANES_D(DLargestFrom) must be less than or equal to "
"(kMaxCapped >> kPow2)");
detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kFactor, Test, -kPow2>::Do(
1, max_lanes);
}
};
template <class Test, int kPow2 = 1>
class ForDemoteVectors {
mutable bool called_ = false;
public:
~ForDemoteVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 475, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMinLanes = size_t{1} << kPow2;
constexpr size_t kMaxCapped = (16 / sizeof(T));
constexpr size_t max_lanes = kMaxCapped;
(void)kMinLanes;
(void)max_lanes;
(void)max_lanes;
detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kMinLanes, Test>::Do(
kMinLanes, max_lanes);
}
};
template <class Test, int kPow2 = 1>
class ForHalfVectors {
mutable bool called_ = false;
public:
~ForHalfVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 512, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMinLanes = size_t{1} << kPow2;
constexpr size_t kMaxCapped = (16 / sizeof(T));
detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kMinLanes, Test>::Do(
kMinLanes, kMaxCapped);
}
};
template <class Test>
class ForPartialVectors {
mutable bool called_ = false;
public:
~ForPartialVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 545, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T t) const {
called_ = true;
ForExtendableVectors<Test, 0>()(t);
}
};
template <class Test>
using ForPartialFixedOrFullScalableVectors = ForPartialVectors<Test>;
template <class Func>
void ForSignedTypes(const Func& func) {
func(int8_t());
func(int16_t());
func(int32_t());
func(int64_t());
}
template <class Func>
void ForUnsignedTypes(const Func& func) {
func(uint8_t());
func(uint16_t());
func(uint32_t());
func(uint64_t());
}
template <class Func>
void ForIntegerTypes(const Func& func) {
ForSignedTypes(func);
ForUnsignedTypes(func);
}
template <class Func>
void ForFloatTypes(const Func& func) {
func(float());
func(double());
}
template <class Func>
void ForAllTypes(const Func& func) {
ForIntegerTypes(func);
ForFloatTypes(func);
}
template <class Func>
void ForUI8(const Func& func) {
func(uint8_t());
func(int8_t());
}
template <class Func>
void ForUI16(const Func& func) {
func(uint16_t());
func(int16_t());
}
template <class Func>
void ForUIF16(const Func& func) {
ForUI16(func);
func(float16_t());
}
template <class Func>
void ForUI32(const Func& func) {
func(uint32_t());
func(int32_t());
}
template <class Func>
void ForUIF32(const Func& func) {
ForUI32(func);
func(float());
}
template <class Func>
void ForUI64(const Func& func) {
func(uint64_t());
func(int64_t());
}
template <class Func>
void ForUIF64(const Func& func) {
ForUI64(func);
func(double());
}
template <class Func>
void ForUI3264(const Func& func) {
ForUI32(func);
ForUI64(func);
}
template <class Func>
void ForUIF3264(const Func& func) {
ForUIF32(func);
ForUIF64(func);
}
template <class Func>
void ForU163264(const Func& func) {
func(uint16_t());
func(uint32_t());
func(uint64_t());
}
template <class Func>
void ForUI163264(const Func& func) {
ForUI16(func);
ForUI3264(func);
}
template <class Func>
void ForUIF163264(const Func& func) {
ForUIF16(func);
ForUIF3264(func);
}
constexpr size_t AdjustedReps(size_t max_reps) {
return ((max_reps) > (2) ? (max_reps) : (2));
}
constexpr size_t AdjustedLog2Reps(size_t max_pow2) {
return max_pow2;
}
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC push_options
#pragma GCC target "sse2,ssse3"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSSE3 {
template <class Out, class In>
inline Out BitCast(const In& in) {
static_assert(sizeof(Out) == sizeof(In), "");
Out out;
CopyBytes<sizeof(out)>(&in, &out);
return out;
}
template <class T, class D>
__attribute__((noinline)) void TestMath(const char* name, T (*fx1)(T),
Vec<D> (*fxN)(D, VecArg<Vec<D>>), D d, T min, T max,
uint64_t max_error_ulp) {
using UintT = MakeUnsigned<T>;
const UintT min_bits = BitCast<UintT>(min);
const UintT max_bits = BitCast<UintT>(max);
int range_count = 1;
UintT ranges[2][2] = {{min_bits, max_bits}, {0, 0}};
if ((min < 0.0) && (max > 0.0)) {
ranges[0][0] = BitCast<UintT>(static_cast<T>(+0.0));
ranges[0][1] = max_bits;
ranges[1][0] = BitCast<UintT>(static_cast<T>(-0.0));
ranges[1][1] = min_bits;
range_count = 2;
}
uint64_t max_ulp = 0;
constexpr UintT kSamplesPerRange = static_cast<UintT>(AdjustedReps(4000));
for (int range_index = 0; range_index < range_count; ++range_index) {
const UintT start = ranges[range_index][0];
const UintT stop = ranges[range_index][1];
const UintT step = ((1) > (((stop - start) / kSamplesPerRange)) ? (1) : (((stop - start) / kSamplesPerRange)));
for (UintT value_bits = start; value_bits <= stop; value_bits += step) {
const T value = BitCast<T>(((((start) > (value_bits) ? (start) : (value_bits))) < (stop) ? (((start) > (value_bits) ? (start) : (value_bits))) : (stop)));
const T actual = GetLane(fxN(d, Set(d, value)));
const T expected = fx1(value);
const auto ulp = hwy::detail::ComputeUlpDelta(actual, expected);
max_ulp = ((max_ulp) > (ulp) ? (max_ulp) : (ulp));
if (ulp > max_error_ulp) {
fprintf(
stderr
, "KO %s: %s(%.17g) expected %.17g actual %.17g ulp %g max ulp %u\n",
hwy::TypeName(T(), Lanes(d)).c_str(), name, value, expected,
actual, static_cast<double>(ulp),
static_cast<uint32_t>(max_error_ulp));
if( value == 5.526720574244119e-20 && expected == 5.526720574244119e-20 && actual == 0) exit(42);
}
else {
fprintf(
stderr
, "OK %s: %s(%.17g) expected %.17g actual %.17g ulp %g max ulp %u\n",
hwy::TypeName(T(), Lanes(d)).c_str(), name, value, expected,
actual, static_cast<double>(ulp),
static_cast<uint32_t>(max_error_ulp));
}
}
}
fprintf(
stderr
, "%s: %s max_ulp %g\n", hwy::TypeName(T(), Lanes(d)).c_str(),
name, static_cast<double>(max_ulp));
}
const float kNearOneF = BitCast<float>(0x3F7FFFFF);
const double kNearOneD = BitCast<double>(0x3FEFFFFFFFFFFFFFULL);
struct TestLog1p { template <class T, class D> __attribute__((noinline)) void operator()(T, D d) { if (sizeof(T) == 4) { TestMath<T, D>("Log1p", std::log1p, CallLog1p, d, +0.0f, +1e37f, 3); } else { TestMath<T, D>("Log1p", std::log1p, CallLog1p, d, static_cast<T>(+0.0), static_cast<T>(+double(1.79769313486231570814527423731704357e+308L)
), 2); } } }; __attribute__((noinline)) void TestAllLog1p() { ForFloatTypes(ForPartialVectors<TestLog1p>()); }
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wuninitialized"
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#pragma GCC push_options
#pragma GCC target("sse4.1")
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_testz_si128 (__m128i __M, __m128i __V)
{
return __builtin_ia32_ptestz128 ((__v2di)__M, (__v2di)__V);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_testc_si128 (__m128i __M, __m128i __V)
{
return __builtin_ia32_ptestc128 ((__v2di)__M, (__v2di)__V);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_testnzc_si128 (__m128i __M, __m128i __V)
{
return __builtin_ia32_ptestnzc128 ((__v2di)__M, (__v2di)__V);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_round_pd (__m128d __V, const int __M)
{
return (__m128d) __builtin_ia32_roundpd ((__v2df)__V, __M);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_round_sd(__m128d __D, __m128d __V, const int __M)
{
return (__m128d) __builtin_ia32_roundsd ((__v2df)__D,
(__v2df)__V,
__M);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_round_ps (__m128 __V, const int __M)
{
return (__m128) __builtin_ia32_roundps ((__v4sf)__V, __M);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_round_ss (__m128 __D, __m128 __V, const int __M)
{
return (__m128) __builtin_ia32_roundss ((__v4sf)__D,
(__v4sf)__V,
__M);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_blend_epi16 (__m128i __X, __m128i __Y, const int __M)
{
return (__m128i) __builtin_ia32_pblendw128 ((__v8hi)__X,
(__v8hi)__Y,
__M);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_blendv_epi8 (__m128i __X, __m128i __Y, __m128i __M)
{
return (__m128i) __builtin_ia32_pblendvb128 ((__v16qi)__X,
(__v16qi)__Y,
(__v16qi)__M);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_blend_ps (__m128 __X, __m128 __Y, const int __M)
{
return (__m128) __builtin_ia32_blendps ((__v4sf)__X,
(__v4sf)__Y,
__M);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_blendv_ps (__m128 __X, __m128 __Y, __m128 __M)
{
return (__m128) __builtin_ia32_blendvps ((__v4sf)__X,
(__v4sf)__Y,
(__v4sf)__M);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_blend_pd (__m128d __X, __m128d __Y, const int __M)
{
return (__m128d) __builtin_ia32_blendpd ((__v2df)__X,
(__v2df)__Y,
__M);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_blendv_pd (__m128d __X, __m128d __Y, __m128d __M)
{
return (__m128d) __builtin_ia32_blendvpd ((__v2df)__X,
(__v2df)__Y,
(__v2df)__M);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_dp_ps (__m128 __X, __m128 __Y, const int __M)
{
return (__m128) __builtin_ia32_dpps ((__v4sf)__X,
(__v4sf)__Y,
__M);
}
extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_dp_pd (__m128d __X, __m128d __Y, const int __M)
{
return (__m128d) __builtin_ia32_dppd ((__v2df)__X,
(__v2df)__Y,
__M);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpeq_epi64 (__m128i __X, __m128i __Y)
{
return (__m128i) ((__v2di)__X == (__v2di)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_min_epi8 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_pminsb128 ((__v16qi)__X, (__v16qi)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_max_epi8 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_pmaxsb128 ((__v16qi)__X, (__v16qi)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_min_epu16 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_pminuw128 ((__v8hi)__X, (__v8hi)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_max_epu16 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_pmaxuw128 ((__v8hi)__X, (__v8hi)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_min_epi32 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_pminsd128 ((__v4si)__X, (__v4si)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_max_epi32 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_pmaxsd128 ((__v4si)__X, (__v4si)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_min_epu32 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_pminud128 ((__v4si)__X, (__v4si)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_max_epu32 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_pmaxud128 ((__v4si)__X, (__v4si)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mullo_epi32 (__m128i __X, __m128i __Y)
{
return (__m128i) ((__v4su)__X * (__v4su)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mul_epi32 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_pmuldq128 ((__v4si)__X, (__v4si)__Y);
}
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_insert_ps (__m128 __D, __m128 __S, const int __N)
{
return (__m128) __builtin_ia32_insertps128 ((__v4sf)__D,
(__v4sf)__S,
__N);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_extract_ps (__m128 __X, const int __N)
{
union { int __i; float __f; } __tmp;
__tmp.__f = __builtin_ia32_vec_ext_v4sf ((__v4sf)__X, __N);
return __tmp.__i;
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_insert_epi8 (__m128i __D, int __S, const int __N)
{
return (__m128i) __builtin_ia32_vec_set_v16qi ((__v16qi)__D,
__S, __N);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_insert_epi32 (__m128i __D, int __S, const int __N)
{
return (__m128i) __builtin_ia32_vec_set_v4si ((__v4si)__D,
__S, __N);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_extract_epi8 (__m128i __X, const int __N)
{
return (unsigned char) __builtin_ia32_vec_ext_v16qi ((__v16qi)__X, __N);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_extract_epi32 (__m128i __X, const int __N)
{
return __builtin_ia32_vec_ext_v4si ((__v4si)__X, __N);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_minpos_epu16 (__m128i __X)
{
return (__m128i) __builtin_ia32_phminposuw128 ((__v8hi)__X);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtepi8_epi32 (__m128i __X)
{
return (__m128i) __builtin_ia32_pmovsxbd128 ((__v16qi)__X);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtepi16_epi32 (__m128i __X)
{
return (__m128i) __builtin_ia32_pmovsxwd128 ((__v8hi)__X);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtepi8_epi64 (__m128i __X)
{
return (__m128i) __builtin_ia32_pmovsxbq128 ((__v16qi)__X);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtepi32_epi64 (__m128i __X)
{
return (__m128i) __builtin_ia32_pmovsxdq128 ((__v4si)__X);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtepi16_epi64 (__m128i __X)
{
return (__m128i) __builtin_ia32_pmovsxwq128 ((__v8hi)__X);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtepi8_epi16 (__m128i __X)
{
return (__m128i) __builtin_ia32_pmovsxbw128 ((__v16qi)__X);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtepu8_epi32 (__m128i __X)
{
return (__m128i) __builtin_ia32_pmovzxbd128 ((__v16qi)__X);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtepu16_epi32 (__m128i __X)
{
return (__m128i) __builtin_ia32_pmovzxwd128 ((__v8hi)__X);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtepu8_epi64 (__m128i __X)
{
return (__m128i) __builtin_ia32_pmovzxbq128 ((__v16qi)__X);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtepu32_epi64 (__m128i __X)
{
return (__m128i) __builtin_ia32_pmovzxdq128 ((__v4si)__X);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtepu16_epi64 (__m128i __X)
{
return (__m128i) __builtin_ia32_pmovzxwq128 ((__v8hi)__X);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cvtepu8_epi16 (__m128i __X)
{
return (__m128i) __builtin_ia32_pmovzxbw128 ((__v16qi)__X);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_packus_epi32 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_packusdw128 ((__v4si)__X, (__v4si)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_mpsadbw_epu8 (__m128i __X, __m128i __Y, const int __M)
{
return (__m128i) __builtin_ia32_mpsadbw128 ((__v16qi)__X,
(__v16qi)__Y, __M);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_stream_load_si128 (__m128i *__X)
{
return (__m128i) __builtin_ia32_movntdqa ((__v2di *) __X);
}
#pragma GCC push_options
#pragma GCC target("sse4.2")
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpistrm (__m128i __X, __m128i __Y, const int __M)
{
return (__m128i) __builtin_ia32_pcmpistrm128 ((__v16qi)__X,
(__v16qi)__Y,
__M);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpistri (__m128i __X, __m128i __Y, const int __M)
{
return __builtin_ia32_pcmpistri128 ((__v16qi)__X,
(__v16qi)__Y,
__M);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpestrm (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M)
{
return (__m128i) __builtin_ia32_pcmpestrm128 ((__v16qi)__X, __LX,
(__v16qi)__Y, __LY,
__M);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpestri (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M)
{
return __builtin_ia32_pcmpestri128 ((__v16qi)__X, __LX,
(__v16qi)__Y, __LY,
__M);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpistra (__m128i __X, __m128i __Y, const int __M)
{
return __builtin_ia32_pcmpistria128 ((__v16qi)__X,
(__v16qi)__Y,
__M);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpistrc (__m128i __X, __m128i __Y, const int __M)
{
return __builtin_ia32_pcmpistric128 ((__v16qi)__X,
(__v16qi)__Y,
__M);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpistro (__m128i __X, __m128i __Y, const int __M)
{
return __builtin_ia32_pcmpistrio128 ((__v16qi)__X,
(__v16qi)__Y,
__M);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpistrs (__m128i __X, __m128i __Y, const int __M)
{
return __builtin_ia32_pcmpistris128 ((__v16qi)__X,
(__v16qi)__Y,
__M);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpistrz (__m128i __X, __m128i __Y, const int __M)
{
return __builtin_ia32_pcmpistriz128 ((__v16qi)__X,
(__v16qi)__Y,
__M);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpestra (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M)
{
return __builtin_ia32_pcmpestria128 ((__v16qi)__X, __LX,
(__v16qi)__Y, __LY,
__M);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpestrc (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M)
{
return __builtin_ia32_pcmpestric128 ((__v16qi)__X, __LX,
(__v16qi)__Y, __LY,
__M);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpestro (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M)
{
return __builtin_ia32_pcmpestrio128 ((__v16qi)__X, __LX,
(__v16qi)__Y, __LY,
__M);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpestrs (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M)
{
return __builtin_ia32_pcmpestris128 ((__v16qi)__X, __LX,
(__v16qi)__Y, __LY,
__M);
}
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpestrz (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M)
{
return __builtin_ia32_pcmpestriz128 ((__v16qi)__X, __LX,
(__v16qi)__Y, __LY,
__M);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_cmpgt_epi64 (__m128i __X, __m128i __Y)
{
return (__m128i) ((__v2di)__X > (__v2di)__Y);
}
#pragma GCC pop_options
#pragma GCC pop_options
#pragma GCC push_options
#pragma GCC target("popcnt")
extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_popcnt_u32 (unsigned int __X)
{
return __builtin_popcount (__X);
}
#pragma GCC pop_options
#pragma GCC push_options
#pragma GCC target("crc32")
extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_crc32_u8 (unsigned int __C, unsigned char __V)
{
return __builtin_ia32_crc32qi (__C, __V);
}
extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_crc32_u16 (unsigned int __C, unsigned short __V)
{
return __builtin_ia32_crc32hi (__C, __V);
}
extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_crc32_u32 (unsigned int __C, unsigned int __V)
{
return __builtin_ia32_crc32si (__C, __V);
}
#pragma GCC pop_options
#pragma GCC push_options
#pragma GCC target("aes,sse2")
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_aesdec_si128 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_aesdec128 ((__v2di)__X, (__v2di)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_aesdeclast_si128 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_aesdeclast128 ((__v2di)__X,
(__v2di)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_aesenc_si128 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_aesenc128 ((__v2di)__X, (__v2di)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_aesenclast_si128 (__m128i __X, __m128i __Y)
{
return (__m128i) __builtin_ia32_aesenclast128 ((__v2di)__X, (__v2di)__Y);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_aesimc_si128 (__m128i __X)
{
return (__m128i) __builtin_ia32_aesimc128 ((__v2di)__X);
}
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_aeskeygenassist_si128 (__m128i __X, const int __C)
{
return (__m128i) __builtin_ia32_aeskeygenassist128 ((__v2di)__X, __C);
}
#pragma GCC pop_options
#pragma GCC push_options
#pragma GCC target("pclmul,sse2")
extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
_mm_clmulepi64_si128 (__m128i __X, __m128i __Y, const int __I)
{
return (__m128i) __builtin_ia32_pclmulqdq128 ((__v2di)__X,
(__v2di)__Y, __I);
}
#pragma GCC pop_options
#pragma GCC push_options
#pragma GCC target "sse2,ssse3" ",sse4.1,sse4.2" ",pclmul,aes"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSE4 {
template <class V>
using VecArg = V;
namespace detail {
constexpr size_t ScaleByPower(size_t N, int pow2) {
return pow2 >= 0 ? (N << pow2) : (N >> (-pow2));
}
template <typename T>
inline __attribute__((always_inline)) void MaybeUnpoison(T* __restrict__ unaligned, size_t count) {
(void)unaligned;
(void)count;
}
}
template <typename Lane, size_t N, int kPow2>
struct Simd {
constexpr Simd() = default;
using T = Lane;
private:
static_assert(sizeof(Lane) <= 8, "Lanes are up to 64-bit");
static constexpr size_t kWhole = N & 0xFFFFF;
static constexpr int kFrac = static_cast<int>(N >> 20);
static_assert(kWhole <= 8 * 65536 && kFrac <= 3, "Out of range");
static_assert(kFrac == 0 || kWhole == 1, "If frac, whole must be 1");
static_assert((kWhole & (kWhole - 1)) == 0 && kWhole != 0, "Not 2^x");
static_assert(kPow2 >= -8, "Forgot kPow2 recursion terminator?");
public:
static constexpr size_t kPrivateLanes =
((size_t{1}) > (detail::ScaleByPower(kWhole, kPow2 - kFrac)) ? (size_t{1}) : (detail::ScaleByPower(kWhole, kPow2 - kFrac)));
constexpr size_t MaxLanes() const { return kPrivateLanes; }
constexpr size_t MaxBytes() const { return kPrivateLanes * sizeof(Lane); }
constexpr size_t MaxBlocks() const { return (MaxBytes() + 15) / 16; }
constexpr int Pow2() const { return kPow2; }
template <typename NewT>
static constexpr size_t RepartitionLanes() {
return (kPrivateLanes * sizeof(T) + sizeof(NewT) - 1) / sizeof(NewT);
}
template <typename NewT>
static constexpr int RebindPow2() {
return kPow2 +
((sizeof(NewT) >= sizeof(T))
? static_cast<int>(CeilLog2(sizeof(NewT) / sizeof(T)))
: -static_cast<int>(CeilLog2(sizeof(T) / sizeof(NewT))));
}
private:
template <int kNewPow2, size_t kNewMaxLanes>
static constexpr size_t WholeN() {
return detail::ScaleByPower(kNewMaxLanes, -kNewPow2);
}
template <int kNewPow2, size_t kNewMaxLanes>
static constexpr size_t FracN() {
static_assert(65536 <= (size_t{1} << 20), "Change bit shift");
return static_cast<size_t>(
1 + (((0) > (kNewPow2 - static_cast<int>(CeilLog2(kNewMaxLanes))) ? (0) : (kNewPow2 - static_cast<int>(CeilLog2(kNewMaxLanes))))
<< 20));
}
public:
template <int kNewPow2, size_t kNewMaxLanes>
static constexpr size_t NewN() {
return WholeN<kNewPow2, kNewMaxLanes>() == 0
? FracN<kNewPow2, kNewMaxLanes>()
: WholeN<kNewPow2, kNewMaxLanes>();
}
template <typename NewT>
using Rebind =
Simd<NewT, NewN<RebindPow2<NewT>(), kPrivateLanes>(), RebindPow2<NewT>()>;
template <typename NewT>
using Repartition =
Simd<NewT, NewN<kPow2, RepartitionLanes<NewT>()>(), kPow2>;
using Half = Simd<T, N, kPow2 - 1>;
using Twice = Simd<T, N, kPow2 + 1>;
};
namespace detail {
template <typename T, size_t N, int kPow2>
constexpr bool IsFull(Simd<T, N, kPow2> ) {
return N == (16 / sizeof(T)) && kPow2 == 0;
}
template <typename T, size_t N, int kPow2>
struct ClampNAndPow2 {
using type = Simd<T, ((N) < (65536) ? (N) : (65536)), ((kPow2) < (3) ? (kPow2) : (3))>;
};
template <typename T, int kPow2>
struct ScalableTagChecker {
using type = typename ClampNAndPow2<T, (16 / sizeof(T)), kPow2>::type;
};
template <typename T, size_t kLimit, int kPow2>
struct CappedTagChecker {
static_assert(kLimit != 0, "Does not make sense to have zero lanes");
static constexpr size_t kLimitPow2 = size_t{1} << hwy::FloorLog2(kLimit);
static constexpr size_t N = ((kLimitPow2) < ((16 / sizeof(T))) ? (kLimitPow2) : ((16 / sizeof(T))));
using type = typename ClampNAndPow2<T, N, kPow2>::type;
};
template <typename T, size_t kNumLanes>
struct FixedTagChecker {
static_assert(kNumLanes != 0, "Does not make sense to have zero lanes");
static_assert(kNumLanes <= (16 / sizeof(T)), "Too many lanes");
using type = Simd<T, kNumLanes, 0>;
};
}
template <typename T, int kPow2 = 0>
using ScalableTag = typename detail::ScalableTagChecker<T, kPow2>::type;
template <typename T, size_t kLimit, int kPow2 = 0>
using CappedTag = typename detail::CappedTagChecker<T, kLimit, kPow2>::type;
template <typename T, size_t kLimit, int kPow2 = 0>
using CappedTagIfFixed = CappedTag<T, kLimit, kPow2>;
template <typename T, size_t kNumLanes>
using FixedTag = typename detail::FixedTagChecker<T, kNumLanes>::type;
template <typename T>
using Full16 = Simd<T, 2 / sizeof(T), 0>;
template <typename T>
using Full32 = Simd<T, 4 / sizeof(T), 0>;
template <typename T>
using Full64 = Simd<T, 8 / sizeof(T), 0>;
template <typename T>
using Full128 = Simd<T, 16 / sizeof(T), 0>;
template <class D>
using TFromD = typename D::T;
template <class D>
inline __attribute__((always_inline)) __attribute__((unused)) constexpr size_t MaxLanes(D) {
return D::kPrivateLanes;
}
template <class D>
inline __attribute__((always_inline)) __attribute__((unused)) constexpr size_t Lanes(D) {
return D::kPrivateLanes;
}
template <class T, class D>
using Rebind = typename D::template Rebind<T>;
template <class D>
using RebindToSigned = Rebind<MakeSigned<TFromD<D>>, D>;
template <class D>
using RebindToUnsigned = Rebind<MakeUnsigned<TFromD<D>>, D>;
template <class D>
using RebindToFloat = Rebind<MakeFloat<TFromD<D>>, D>;
template <class T, class D>
using Repartition = typename D::template Repartition<T>;
template <class D>
using RepartitionToWide = Repartition<MakeWide<TFromD<D>>, D>;
template <class D>
using RepartitionToNarrow = Repartition<MakeNarrow<TFromD<D>>, D>;
template <class D>
using Half = typename D::Half;
template <class D>
using Twice = typename D::Twice;
template <class D>
using BlockDFromD =
Simd<TFromD<D>, ((16 / sizeof(TFromD<D>)) < (D::kPrivateLanes) ? (16 / sizeof(TFromD<D>)) : (D::kPrivateLanes)), 0>;
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC push_options
#pragma GCC target "sse2,ssse3" ",sse4.1,sse4.2" ",pclmul,aes"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSE4 {
namespace detail {
template <typename T>
struct Raw128 {
using type = __m128i;
};
template <>
struct Raw128<float> {
using type = __m128;
};
template <>
struct Raw128<double> {
using type = __m128d;
};
}
template <typename T, size_t N = 16 / sizeof(T)>
class Vec128 {
using Raw = typename detail::Raw128<T>::type;
public:
using PrivateT = T;
static constexpr size_t kPrivateN = N;
inline __attribute__((always_inline)) Vec128& operator*=(const Vec128 other) {
return *this = (*this * other);
}
inline __attribute__((always_inline)) Vec128& operator/=(const Vec128 other) {
return *this = (*this / other);
}
inline __attribute__((always_inline)) Vec128& operator+=(const Vec128 other) {
return *this = (*this + other);
}
inline __attribute__((always_inline)) Vec128& operator-=(const Vec128 other) {
return *this = (*this - other);
}
inline __attribute__((always_inline)) Vec128& operator&=(const Vec128 other) {
return *this = (*this & other);
}
inline __attribute__((always_inline)) Vec128& operator|=(const Vec128 other) {
return *this = (*this | other);
}
inline __attribute__((always_inline)) Vec128& operator^=(const Vec128 other) {
return *this = (*this ^ other);
}
Raw raw;
};
template <typename T>
using Vec64 = Vec128<T, 8 / sizeof(T)>;
template <typename T>
using Vec32 = Vec128<T, 4 / sizeof(T)>;
template <typename T>
using Vec16 = Vec128<T, 2 / sizeof(T)>;
template <typename T, size_t N = 16 / sizeof(T)>
struct Mask128 {
typename detail::Raw128<T>::type raw;
};
namespace detail {
template <typename T, size_t N>
constexpr uint64_t OnlyActive(uint64_t mask_bits) {
return ((N * sizeof(T)) == 16) ? mask_bits : mask_bits & ((1ull << N) - 1);
}
}
template <class V>
using DFromV = Simd<typename V::PrivateT, V::kPrivateN, 0>;
template <class V>
using TFromV = typename V::PrivateT;
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<!hwy::IsFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<TFromD<D>, D::kPrivateLanes> Zero(D ) {
return Vec128<TFromD<D>, D::kPrivateLanes>{_mm_setzero_si128()};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, D::kPrivateLanes> Zero(D ) {
return Vec128<float, D::kPrivateLanes>{_mm_setzero_ps()};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, D::kPrivateLanes> Zero(D ) {
return Vec128<double, D::kPrivateLanes>{_mm_setzero_pd()};
}
template <class D>
using VFromD = decltype(Zero(D()));
template <class D>
struct Vec2 {
VFromD<D> v0;
VFromD<D> v1;
};
template <class D>
struct Vec3 {
VFromD<D> v0;
VFromD<D> v1;
VFromD<D> v2;
};
template <class D>
struct Vec4 {
VFromD<D> v0;
VFromD<D> v1;
VFromD<D> v2;
VFromD<D> v3;
};
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec2<D> Create2(D , VFromD<D> v0, VFromD<D> v1) {
return Vec2<D>{v0, v1};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec3<D> Create3(D , VFromD<D> v0, VFromD<D> v1, VFromD<D> v2) {
return Vec3<D>{v0, v1, v2};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec4<D> Create4(D , VFromD<D> v0, VFromD<D> v1, VFromD<D> v2,
VFromD<D> v3) {
return Vec4<D>{v0, v1, v2, v3};
}
template <size_t kIndex, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Get2(Vec2<D> tuple) {
static_assert(kIndex < 2, "Tuple index out of bounds");
return kIndex == 0 ? tuple.v0 : tuple.v1;
}
template <size_t kIndex, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Get3(Vec3<D> tuple) {
static_assert(kIndex < 3, "Tuple index out of bounds");
return kIndex == 0 ? tuple.v0 : kIndex == 1 ? tuple.v1 : tuple.v2;
}
template <size_t kIndex, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Get4(Vec4<D> tuple) {
static_assert(kIndex < 4, "Tuple index out of bounds");
return kIndex == 0 ? tuple.v0
: kIndex == 1 ? tuple.v1
: kIndex == 2 ? tuple.v2
: tuple.v3;
}
namespace detail {
inline __attribute__((always_inline)) __m128i BitCastToInteger(__m128i v) { return v; }
inline __attribute__((always_inline)) __m128i BitCastToInteger(__m128 v) { return _mm_castps_si128(v); }
inline __attribute__((always_inline)) __m128i BitCastToInteger(__m128d v) { return _mm_castpd_si128(v); }
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<uint8_t, N * sizeof(T)> BitCastToByte(Vec128<T, N> v) {
return Vec128<uint8_t, N * sizeof(T)>{BitCastToInteger(v.raw)};
}
template <typename T>
struct BitCastFromInteger128 {
inline __attribute__((always_inline)) __m128i operator()(__m128i v) { return v; }
};
template <>
struct BitCastFromInteger128<float> {
inline __attribute__((always_inline)) __m128 operator()(__m128i v) { return _mm_castsi128_ps(v); }
};
template <>
struct BitCastFromInteger128<double> {
inline __attribute__((always_inline)) __m128d operator()(__m128i v) { return _mm_castsi128_pd(v); }
};
template <class D>
inline __attribute__((always_inline)) VFromD<D> BitCastFromByte(D ,
Vec128<uint8_t, D().MaxBytes()> v) {
return VFromD<D>{BitCastFromInteger128<TFromD<D>>()(v.raw)};
}
}
template <class D, typename FromT>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> BitCast(D d,
Vec128<FromT, Repartition<FromT, D>().MaxLanes()> v) {
return detail::BitCastFromByte(d, detail::BitCastToByte(v));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , TFromD<D> t) {
return VFromD<D>{_mm_set1_epi8(static_cast<char>(t))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr,
hwy::EnableIf<!hwy::IsSpecialFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , TFromD<D> t) {
return VFromD<D>{_mm_set1_epi16(static_cast<short>(t))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>() || IsSame<TFromD<D>, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , TFromD<D> t) {
return VFromD<D>{_mm_set1_epi32(static_cast<int>(t))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>() || IsSame<TFromD<D>, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , TFromD<D> t) {
return VFromD<D>{_mm_set1_epi64x(static_cast<long long>(t))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , float t) {
return VFromD<D>{_mm_set1_ps(t)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D , double t) {
return VFromD<D>{_mm_set1_pd(t)};
}
template <class D, hwy::EnableIf<hwy::IsSpecialFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D df, TFromD<D> t) {
const RebindToUnsigned<decltype(df)> du;
static_assert(sizeof(TFromD<D>) == 2, "Expecting [b]f16");
uint16_t bits;
CopyBytes<2>(&t, &bits);
return BitCast(df, Set(du, bits));
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wuninitialized"
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<!hwy::IsFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Undefined(D ) {
return VFromD<D>{_mm_undefined_si128()};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Undefined(D ) {
return VFromD<D>{_mm_undefined_ps()};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Undefined(D ) {
return VFromD<D>{_mm_undefined_pd()};
}
#pragma GCC diagnostic pop
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T GetLane(const Vec128<T, N> v) {
return static_cast<T>(_mm_cvtsi128_si32(v.raw) & 0xFF);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T GetLane(const Vec128<T, N> v) {
return static_cast<T>(_mm_cvtsi128_si32(v.raw) & 0xFFFF);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T GetLane(const Vec128<T, N> v) {
return static_cast<T>(_mm_cvtsi128_si32(v.raw));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) float GetLane(const Vec128<float, N> v) {
return _mm_cvtss_f32(v.raw);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T GetLane(const Vec128<T, N> v) {
const DFromV<decltype(v)> d;
alignas(16) T lanes[2];
Store(v, d, lanes);
return lanes[0];
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) double GetLane(const Vec128<double, N> v) {
return _mm_cvtsd_f64(v.raw);
}
template <class D, class FromV, hwy::EnableIf<DFromV<FromV>::kPrivateLanes * sizeof(TFromV<FromV>) <= 16>* = nullptr,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ResizeBitCast(D d, FromV v) {
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, VFromD<decltype(du8)>{detail::BitCastToInteger(v.raw)});
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> And(Vec128<T, N> a, Vec128<T, N> b) {
return Vec128<T, N>{_mm_and_si128(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> And(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_and_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> And(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_and_pd(a.raw, b.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> AndNot(Vec128<T, N> not_mask, Vec128<T, N> mask) {
return Vec128<T, N>{_mm_andnot_si128(not_mask.raw, mask.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> AndNot(const Vec128<float, N> not_mask,
const Vec128<float, N> mask) {
return Vec128<float, N>{_mm_andnot_ps(not_mask.raw, mask.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> AndNot(const Vec128<double, N> not_mask,
const Vec128<double, N> mask) {
return Vec128<double, N>{_mm_andnot_pd(not_mask.raw, mask.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Or(Vec128<T, N> a, Vec128<T, N> b) {
return Vec128<T, N>{_mm_or_si128(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Or(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_or_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Or(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_or_pd(a.raw, b.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Xor(Vec128<T, N> a, Vec128<T, N> b) {
return Vec128<T, N>{_mm_xor_si128(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Xor(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_xor_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Xor(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_xor_pd(a.raw, b.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Not(const Vec128<T, N> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
using VU = VFromD<decltype(du)>;
return Xor(v, BitCast(d, VU{_mm_set1_epi32(-1)}));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Xor3(Vec128<T, N> x1, Vec128<T, N> x2, Vec128<T, N> x3) {
return Xor(x1, Xor(x2, x3));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Or3(Vec128<T, N> o1, Vec128<T, N> o2, Vec128<T, N> o3) {
return Or(o1, Or(o2, o3));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> OrAnd(Vec128<T, N> o, Vec128<T, N> a1, Vec128<T, N> a2) {
return Or(o, And(a1, a2));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfVecThenElse(Vec128<T, N> mask, Vec128<T, N> yes,
Vec128<T, N> no) {
return IfThenElse(MaskFromVec(mask), yes, no);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator&(const Vec128<T, N> a, const Vec128<T, N> b) {
return And(a, b);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator|(const Vec128<T, N> a, const Vec128<T, N> b) {
return Or(a, b);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator^(const Vec128<T, N> a, const Vec128<T, N> b) {
return Xor(a, b);
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Neg(hwy::FloatTag , const Vec128<T, N> v) {
return Xor(v, SignBit(DFromV<decltype(v)>()));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Neg(hwy::NonFloatTag , const Vec128<T, N> v) {
return Zero(DFromV<decltype(v)>()) - v;
}
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Neg(const Vec128<T, N> v) {
return detail::Neg(hwy::IsFloatTag<T>(), v);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Abs(const Vec128<float, N> v) {
const Vec128<int32_t, N> mask{_mm_set1_epi32(0x7FFFFFFF)};
return v & BitCast(DFromV<decltype(v)>(), mask);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Abs(const Vec128<double, N> v) {
const Vec128<int64_t, N> mask{_mm_set1_epi64x(0x7FFFFFFFFFFFFFFFLL)};
return v & BitCast(DFromV<decltype(v)>(), mask);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CopySign(const V magn, const V sign) {
static_assert(IsFloat<TFromV<V>>(), "Only makes sense for floating-point");
const DFromV<decltype(magn)> d;
const auto msb = SignBit(d);
return BitwiseIfThenElse(msb, sign, magn);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CopySignToAbs(const V abs, const V sign) {
const DFromV<decltype(abs)> d;
return OrAnd(abs, SignBit(d), sign);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> MaskFromVec(const Vec128<T, N> v) {
return Mask128<T, N>{v.raw};
}
template <class D>
using MFromD = decltype(MaskFromVec(VFromD<D>()));
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> VecFromMask(const Mask128<T, N> v) {
return Vec128<T, N>{v.raw};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> VecFromMask(D , MFromD<D> v) {
return VFromD<D>{v.raw};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfThenElse(Mask128<T, N> mask, Vec128<T, N> yes,
Vec128<T, N> no) {
return Vec128<T, N>{_mm_blendv_epi8(no.raw, yes.raw, mask.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> IfThenElse(const Mask128<float, N> mask,
const Vec128<float, N> yes,
const Vec128<float, N> no) {
return Vec128<float, N>{_mm_blendv_ps(no.raw, yes.raw, mask.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> IfThenElse(const Mask128<double, N> mask,
const Vec128<double, N> yes,
const Vec128<double, N> no) {
return Vec128<double, N>{_mm_blendv_pd(no.raw, yes.raw, mask.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfThenElseZero(Mask128<T, N> mask, Vec128<T, N> yes) {
return yes & VecFromMask(DFromV<decltype(yes)>(), mask);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfThenZeroElse(Mask128<T, N> mask, Vec128<T, N> no) {
return AndNot(VecFromMask(DFromV<decltype(no)>(), mask), no);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> Not(const Mask128<T, N> m) {
const Simd<T, N, 0> d;
return MaskFromVec(Not(VecFromMask(d, m)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> And(const Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(And(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> AndNot(const Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(AndNot(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> Or(const Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(Or(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> Xor(const Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(Xor(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> ExclusiveNeither(const Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(AndNot(VecFromMask(d, a), Not(VecFromMask(d, b))));
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> ShiftLeft(const Vec128<uint16_t, N> v) {
return Vec128<uint16_t, N>{_mm_slli_epi16(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> ShiftLeft(const Vec128<uint32_t, N> v) {
return Vec128<uint32_t, N>{_mm_slli_epi32(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> ShiftLeft(const Vec128<uint64_t, N> v) {
return Vec128<uint64_t, N>{_mm_slli_epi64(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> ShiftLeft(const Vec128<int16_t, N> v) {
return Vec128<int16_t, N>{_mm_slli_epi16(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> ShiftLeft(const Vec128<int32_t, N> v) {
return Vec128<int32_t, N>{_mm_slli_epi32(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> ShiftLeft(const Vec128<int64_t, N> v) {
return Vec128<int64_t, N>{_mm_slli_epi64(v.raw, kBits)};
}
template <int kBits, typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftLeft(const Vec128<T, N> v) {
const DFromV<decltype(v)> d8;
const Vec128<T, N> shifted{ShiftLeft<kBits>(Vec128<MakeWide<T>>{v.raw}).raw};
return kBits == 1
? (v + v)
: (shifted & Set(d8, static_cast<T>((0xFF << kBits) & 0xFF)));
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> ShiftRight(const Vec128<uint16_t, N> v) {
return Vec128<uint16_t, N>{_mm_srli_epi16(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> ShiftRight(const Vec128<uint32_t, N> v) {
return Vec128<uint32_t, N>{_mm_srli_epi32(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> ShiftRight(const Vec128<uint64_t, N> v) {
return Vec128<uint64_t, N>{_mm_srli_epi64(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> ShiftRight(const Vec128<uint8_t, N> v) {
const DFromV<decltype(v)> d8;
const Vec128<uint8_t, N> shifted{
ShiftRight<kBits>(Vec128<uint16_t>{v.raw}).raw};
return shifted & Set(d8, 0xFF >> kBits);
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> ShiftRight(const Vec128<int16_t, N> v) {
return Vec128<int16_t, N>{_mm_srai_epi16(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> ShiftRight(const Vec128<int32_t, N> v) {
return Vec128<int32_t, N>{_mm_srai_epi32(v.raw, kBits)};
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> ShiftRight(const Vec128<int8_t, N> v) {
const DFromV<decltype(v)> di;
const RebindToUnsigned<decltype(di)> du;
const auto shifted = BitCast(di, ShiftRight<kBits>(BitCast(du, v)));
const auto shifted_sign = BitCast(di, Set(du, 0x80 >> kBits));
return (shifted ^ shifted_sign) - shifted_sign;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Load(D , const T* __restrict__ aligned) {
return Vec128<T>{_mm_load_si128(reinterpret_cast<const __m128i*>(aligned))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> Load(D , const float* __restrict__ aligned) {
return Vec128<float>{_mm_load_ps(aligned)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> Load(D , const double* __restrict__ aligned) {
return Vec128<double>{_mm_load_pd(aligned)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> LoadU(D , const T* __restrict__ p) {
return Vec128<T>{_mm_loadu_si128(reinterpret_cast<const __m128i*>(p))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> LoadU(D , const float* __restrict__ p) {
return Vec128<float>{_mm_loadu_ps(p)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> LoadU(D , const double* __restrict__ p) {
return Vec128<double>{_mm_loadu_pd(p)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> Load(D , const T* __restrict__ p) {
return Vec64<T>{_mm_loadl_epi64(reinterpret_cast<const __m128i*>(p))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<float> Load(D , const float* __restrict__ p) {
const __m128 hi = _mm_setzero_ps();
return Vec64<float>{_mm_loadl_pi(hi, reinterpret_cast<const __m64*>(p))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<double> Load(D , const double* __restrict__ p) {
return Vec64<double>{_mm_load_sd(p)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<float> Load(D , const float* __restrict__ p) {
return Vec32<float>{_mm_load_ss(p)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Load(D d, const T* __restrict__ p) {
detail::MaybeUnpoison(p, Lanes(d));
int32_t bits = 0;
CopyBytes<d.MaxBytes()>(p, &bits);
return VFromD<D>{_mm_cvtsi32_si128(bits)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> LoadU(D d, const T* __restrict__ p) {
return Load(d, p);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> LoadDup128(D d, const T* __restrict__ p) {
return LoadU(d, p);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec128<T> v, D , T* __restrict__ aligned) {
_mm_store_si128(reinterpret_cast<__m128i*>(aligned), v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec128<float> v, D , float* __restrict__ aligned) {
_mm_store_ps(aligned, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec128<double> v, D ,
double* __restrict__ aligned) {
_mm_store_pd(aligned, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreU(Vec128<T> v, D , T* __restrict__ p) {
_mm_storeu_si128(reinterpret_cast<__m128i*>(p), v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreU(Vec128<float> v, D , float* __restrict__ p) {
_mm_storeu_ps(p, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreU(Vec128<double> v, D , double* __restrict__ p) {
_mm_storeu_pd(p, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec64<T> v, D , T* __restrict__ p) {
_mm_storel_epi64(reinterpret_cast<__m128i*>(p), v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec64<float> v, D , float* __restrict__ p) {
_mm_storel_pi(reinterpret_cast<__m64*>(p), v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec64<double> v, D , double* __restrict__ p) {
_mm_storel_pd(p, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(VFromD<D> v, D d, T* __restrict__ p) {
CopyBytes<d.MaxBytes()>(&v, p);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(Vec32<float> v, D , float* __restrict__ p) {
_mm_store_ss(p, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreU(VFromD<D> v, D d, T* __restrict__ p) {
Store(v, d, p);
}
template <typename T, size_t N, typename TI, size_t NI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<TI, NI> TableLookupBytes(const Vec128<T, N> bytes,
const Vec128<TI, NI> from) {
return Vec128<TI, NI>{_mm_shuffle_epi8(bytes.raw, from.raw)};
}
template <class V, class VI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VI TableLookupBytesOr0(const V bytes, const VI from) {
return TableLookupBytes(bytes, from);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Shuffle2301(const Vec128<T, N> v) {
static_assert(sizeof(T) == 4, "Only for 32-bit lanes");
static_assert(N == 2 || N == 4, "Does not make sense for N=1");
return Vec128<T, N>{_mm_shuffle_epi32(v.raw, 0xB1)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Shuffle2301(const Vec128<float, N> v) {
static_assert(N == 2 || N == 4, "Does not make sense for N=1");
return Vec128<float, N>{_mm_shuffle_ps(v.raw, v.raw, 0xB1)};
}
namespace detail {
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<T> ShuffleTwo2301(const Vec32<T> a, const Vec32<T> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> d2;
const auto ba = Combine(d2, b, a);
alignas(16) const T kShuffle[8] = {1, 0, 7, 6};
return Vec32<T>{TableLookupBytes(ba, Load(d2, kShuffle)).raw};
}
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ShuffleTwo2301(const Vec64<T> a, const Vec64<T> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> d2;
const auto ba = Combine(d2, b, a);
alignas(16) const T kShuffle[8] = {0x0302, 0x0100, 0x0f0e, 0x0d0c};
return Vec64<T>{TableLookupBytes(ba, Load(d2, kShuffle)).raw};
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ShuffleTwo2301(const Vec128<T> a, const Vec128<T> b) {
const DFromV<decltype(a)> d;
const RebindToFloat<decltype(d)> df;
constexpr int m =
(((
2
) << 6) | ((
3
) << 4) | ((
0
) << 2) | (
1
))
;
return BitCast(d, Vec128<float>{_mm_shuffle_ps(BitCast(df, a).raw,
BitCast(df, b).raw, m)});
}
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<T> ShuffleTwo1230(const Vec32<T> a, const Vec32<T> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> d2;
const auto ba = Combine(d2, b, a);
alignas(16) const T kShuffle[8] = {0, 3, 6, 5};
return Vec32<T>{TableLookupBytes(ba, Load(d2, kShuffle)).raw};
}
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ShuffleTwo1230(const Vec64<T> a, const Vec64<T> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> d2;
const auto ba = Combine(d2, b, a);
alignas(16) const T kShuffle[8] = {0x0100, 0x0706, 0x0d0c, 0x0b0a};
return Vec64<T>{TableLookupBytes(ba, Load(d2, kShuffle)).raw};
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ShuffleTwo1230(const Vec128<T> a, const Vec128<T> b) {
const DFromV<decltype(a)> d;
const RebindToFloat<decltype(d)> df;
constexpr int m =
(((
1
) << 6) | ((
2
) << 4) | ((
3
) << 2) | (
0
))
;
return BitCast(d, Vec128<float>{_mm_shuffle_ps(BitCast(df, a).raw,
BitCast(df, b).raw, m)});
}
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<T> ShuffleTwo3012(const Vec32<T> a, const Vec32<T> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> d2;
const auto ba = Combine(d2, b, a);
alignas(16) const T kShuffle[8] = {2, 1, 4, 7};
return Vec32<T>{TableLookupBytes(ba, Load(d2, kShuffle)).raw};
}
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ShuffleTwo3012(const Vec64<T> a, const Vec64<T> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> d2;
const auto ba = Combine(d2, b, a);
alignas(16) const T kShuffle[8] = {0x0504, 0x0302, 0x0908, 0x0f0e};
return Vec64<T>{TableLookupBytes(ba, Load(d2, kShuffle)).raw};
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ShuffleTwo3012(const Vec128<T> a, const Vec128<T> b) {
const DFromV<decltype(a)> d;
const RebindToFloat<decltype(d)> df;
constexpr int m =
(((
3
) << 6) | ((
0
) << 4) | ((
1
) << 2) | (
2
))
;
return BitCast(d, Vec128<float>{_mm_shuffle_ps(BitCast(df, a).raw,
BitCast(df, b).raw, m)});
}
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> Shuffle1032(const Vec128<uint32_t> v) {
return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x4E)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> Shuffle1032(const Vec128<int32_t> v) {
return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x4E)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> Shuffle1032(const Vec128<float> v) {
return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x4E)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t> Shuffle01(const Vec128<uint64_t> v) {
return Vec128<uint64_t>{_mm_shuffle_epi32(v.raw, 0x4E)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t> Shuffle01(const Vec128<int64_t> v) {
return Vec128<int64_t>{_mm_shuffle_epi32(v.raw, 0x4E)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> Shuffle01(const Vec128<double> v) {
return Vec128<double>{_mm_shuffle_pd(v.raw, v.raw, 1)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> Shuffle0321(const Vec128<uint32_t> v) {
return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x39)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> Shuffle0321(const Vec128<int32_t> v) {
return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x39)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> Shuffle0321(const Vec128<float> v) {
return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x39)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> Shuffle2103(const Vec128<uint32_t> v) {
return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x93)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> Shuffle2103(const Vec128<int32_t> v) {
return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x93)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> Shuffle2103(const Vec128<float> v) {
return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x93)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> Shuffle0123(const Vec128<uint32_t> v) {
return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x1B)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> Shuffle0123(const Vec128<int32_t> v) {
return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x1B)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> Shuffle0123(const Vec128<float> v) {
return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x1B)};
}
template <class DTo, typename TFrom, size_t NFrom>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<DTo> RebindMask(DTo dto, Mask128<TFrom, NFrom> m) {
static_assert(sizeof(TFrom) == sizeof(TFromD<DTo>), "Must have same size");
const Simd<TFrom, NFrom, 0> d;
return MaskFromVec(BitCast(dto, VecFromMask(d, m)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> TestBit(Vec128<T, N> v, Vec128<T, N> bit) {
static_assert(!hwy::IsFloat<T>(), "Only integer vectors supported");
return (v & bit) == bit;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint8_t, N> operator==(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Mask128<uint8_t, N>{_mm_cmpeq_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint16_t, N> operator==(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Mask128<uint16_t, N>{_mm_cmpeq_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint32_t, N> operator==(const Vec128<uint32_t, N> a,
const Vec128<uint32_t, N> b) {
return Mask128<uint32_t, N>{_mm_cmpeq_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint64_t, N> operator==(const Vec128<uint64_t, N> a,
const Vec128<uint64_t, N> b) {
return Mask128<uint64_t, N>{_mm_cmpeq_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int8_t, N> operator==(const Vec128<int8_t, N> a,
const Vec128<int8_t, N> b) {
return Mask128<int8_t, N>{_mm_cmpeq_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int16_t, N> operator==(Vec128<int16_t, N> a,
Vec128<int16_t, N> b) {
return Mask128<int16_t, N>{_mm_cmpeq_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int32_t, N> operator==(const Vec128<int32_t, N> a,
const Vec128<int32_t, N> b) {
return Mask128<int32_t, N>{_mm_cmpeq_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int64_t, N> operator==(const Vec128<int64_t, N> a,
const Vec128<int64_t, N> b) {
const DFromV<decltype(a)> d;
RebindToUnsigned<decltype(d)> du;
return RebindMask(d, BitCast(du, a) == BitCast(du, b));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<float, N> operator==(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Mask128<float, N>{_mm_cmpeq_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<double, N> operator==(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Mask128<double, N>{_mm_cmpeq_pd(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint8_t, N> operator!=(Vec128<uint8_t, N> a,
Vec128<uint8_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint16_t, N> operator!=(Vec128<uint16_t, N> a,
Vec128<uint16_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint32_t, N> operator!=(Vec128<uint32_t, N> a,
Vec128<uint32_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint64_t, N> operator!=(Vec128<uint64_t, N> a,
Vec128<uint64_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int8_t, N> operator!=(Vec128<int8_t, N> a,
Vec128<int8_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int16_t, N> operator!=(Vec128<int16_t, N> a,
Vec128<int16_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int32_t, N> operator!=(Vec128<int32_t, N> a,
Vec128<int32_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<int64_t, N> operator!=(Vec128<int64_t, N> a,
Vec128<int64_t, N> b) {
return Not(a == b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<float, N> operator!=(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Mask128<float, N>{_mm_cmpneq_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<double, N> operator!=(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Mask128<double, N>{_mm_cmpneq_pd(a.raw, b.raw)};
}
namespace detail {
template <size_t N>
inline __attribute__((always_inline)) Mask128<int8_t, N> Gt(hwy::SignedTag , Vec128<int8_t, N> a,
Vec128<int8_t, N> b) {
return Mask128<int8_t, N>{_mm_cmpgt_epi8(a.raw, b.raw)};
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<int16_t, N> Gt(hwy::SignedTag , Vec128<int16_t, N> a,
Vec128<int16_t, N> b) {
return Mask128<int16_t, N>{_mm_cmpgt_epi16(a.raw, b.raw)};
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<int32_t, N> Gt(hwy::SignedTag , Vec128<int32_t, N> a,
Vec128<int32_t, N> b) {
return Mask128<int32_t, N>{_mm_cmpgt_epi32(a.raw, b.raw)};
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<int64_t, N> Gt(hwy::SignedTag ,
const Vec128<int64_t, N> a,
const Vec128<int64_t, N> b) {
return Mask128<int64_t, N>{_mm_cmpgt_epi64(a.raw, b.raw)};
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Mask128<T, N> Gt(hwy::UnsignedTag , Vec128<T, N> a,
Vec128<T, N> b) {
const DFromV<decltype(a)> du;
const RebindToSigned<decltype(du)> di;
const Vec128<T, N> msb = Set(du, (LimitsMax<T>() >> 1) + 1);
const auto sa = BitCast(di, Xor(a, msb));
const auto sb = BitCast(di, Xor(b, msb));
return RebindMask(du, Gt(hwy::SignedTag(), sa, sb));
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<float, N> Gt(hwy::FloatTag , Vec128<float, N> a,
Vec128<float, N> b) {
return Mask128<float, N>{_mm_cmpgt_ps(a.raw, b.raw)};
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<double, N> Gt(hwy::FloatTag , Vec128<double, N> a,
Vec128<double, N> b) {
return Mask128<double, N>{_mm_cmpgt_pd(a.raw, b.raw)};
}
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Mask128<T, N> operator>(Vec128<T, N> a, Vec128<T, N> b) {
return detail::Gt(hwy::TypeTag<T>(), a, b);
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) Mask128<T, N> Ge(hwy::SignedTag tag, Vec128<T, N> a,
Vec128<T, N> b) {
return Not(Gt(tag, b, a));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Mask128<T, N> Ge(hwy::UnsignedTag tag, Vec128<T, N> a,
Vec128<T, N> b) {
return Not(Gt(tag, b, a));
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<float, N> Ge(hwy::FloatTag , Vec128<float, N> a,
Vec128<float, N> b) {
return Mask128<float, N>{_mm_cmpge_ps(a.raw, b.raw)};
}
template <size_t N>
inline __attribute__((always_inline)) Mask128<double, N> Ge(hwy::FloatTag , Vec128<double, N> a,
Vec128<double, N> b) {
return Mask128<double, N>{_mm_cmpge_pd(a.raw, b.raw)};
}
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> operator>=(Vec128<T, N> a, Vec128<T, N> b) {
return detail::Ge(hwy::TypeTag<T>(), a, b);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> operator<(Vec128<T, N> a, Vec128<T, N> b) {
return b > a;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> operator<=(Vec128<T, N> a, Vec128<T, N> b) {
return b >= a;
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{_mm_set_epi8(
static_cast<char>(15), static_cast<char>(14), static_cast<char>(13),
static_cast<char>(12), static_cast<char>(11), static_cast<char>(10),
static_cast<char>(9), static_cast<char>(8), static_cast<char>(7),
static_cast<char>(6), static_cast<char>(5), static_cast<char>(4),
static_cast<char>(3), static_cast<char>(2), static_cast<char>(1),
static_cast<char>(0))};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr,
hwy::EnableIf<!hwy::IsSpecialFloat<TFromD<D> >()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{_mm_set_epi16(int16_t{7}, int16_t{6}, int16_t{5}, int16_t{4},
int16_t{3}, int16_t{2}, int16_t{1},
int16_t{0})};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>() || IsSame<TFromD<D>, int32_t>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{
_mm_set_epi32(int32_t{3}, int32_t{2}, int32_t{1}, int32_t{0})};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>() || IsSame<TFromD<D>, int64_t>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{_mm_set_epi64x(int64_t{1}, int64_t{0})};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{_mm_set_ps(3.0f, 2.0f, 1.0f, 0.0f)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Iota0(D ) {
return VFromD<D>{_mm_set_pd(1.0, 0.0)};
}
}
template <class D, typename T2, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Iota(D d, const T2 first) {
const auto result_iota =
detail::Iota0(d) + Set(d, static_cast<TFromD<D>>(first));
return result_iota;
}
template <class D, class M = MFromD<D>, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) M FirstN(D d, size_t num) {
const RebindToSigned<decltype(d)> di;
using TI = TFromD<decltype(di)>;
return RebindMask(d, detail::Iota0(di) < Set(di, static_cast<TI>(num)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> MaskedLoad(MFromD<D> m, D d, const T* __restrict__ p) {
return IfThenElseZero(m, LoadU(d, p));
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> MaskedLoadOr(VFromD<D> v, MFromD<D> m, D d,
const T* __restrict__ p) {
return IfThenElse(m, LoadU(d, p), v);
}
namespace detail {
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void ScalarMaskedStore(VFromD<D> v, MFromD<D> m, D d,
TFromD<D>* __restrict__ p) {
const RebindToSigned<decltype(d)> di;
using TI = TFromD<decltype(di)>;
alignas(16) TI buf[MaxLanes(d)];
alignas(16) TI mask[MaxLanes(d)];
Store(BitCast(di, v), di, buf);
Store(BitCast(di, VecFromMask(d, m)), di, mask);
for (size_t i = 0; i < MaxLanes(d); ++i) {
if (mask[i]) {
CopySameSize(buf + i, p + i);
}
}
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void BlendedStore(VFromD<D> v, MFromD<D> m, D d,
TFromD<D>* __restrict__ p) {
detail::ScalarMaskedStore(v, m, d, p);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> operator+(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_add_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> operator+(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_add_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> operator+(const Vec128<uint32_t, N> a,
const Vec128<uint32_t, N> b) {
return Vec128<uint32_t, N>{_mm_add_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> operator+(const Vec128<uint64_t, N> a,
const Vec128<uint64_t, N> b) {
return Vec128<uint64_t, N>{_mm_add_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> operator+(const Vec128<int8_t, N> a,
const Vec128<int8_t, N> b) {
return Vec128<int8_t, N>{_mm_add_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> operator+(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_add_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> operator+(const Vec128<int32_t, N> a,
const Vec128<int32_t, N> b) {
return Vec128<int32_t, N>{_mm_add_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> operator+(const Vec128<int64_t, N> a,
const Vec128<int64_t, N> b) {
return Vec128<int64_t, N>{_mm_add_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> operator+(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_add_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> operator+(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_add_pd(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> operator-(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_sub_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> operator-(Vec128<uint16_t, N> a,
Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_sub_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> operator-(const Vec128<uint32_t, N> a,
const Vec128<uint32_t, N> b) {
return Vec128<uint32_t, N>{_mm_sub_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> operator-(const Vec128<uint64_t, N> a,
const Vec128<uint64_t, N> b) {
return Vec128<uint64_t, N>{_mm_sub_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> operator-(const Vec128<int8_t, N> a,
const Vec128<int8_t, N> b) {
return Vec128<int8_t, N>{_mm_sub_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> operator-(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_sub_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> operator-(const Vec128<int32_t, N> a,
const Vec128<int32_t, N> b) {
return Vec128<int32_t, N>{_mm_sub_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> operator-(const Vec128<int64_t, N> a,
const Vec128<int64_t, N> b) {
return Vec128<int64_t, N>{_mm_sub_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> operator-(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_sub_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> operator-(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_sub_pd(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N / 8> SumsOf8(const Vec128<uint8_t, N> v) {
return Vec128<uint64_t, N / 8>{_mm_sad_epu8(v.raw, _mm_setzero_si128())};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N / 8> SumsOf8AbsDiff(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint64_t, N / 8>{_mm_sad_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> SaturatedAdd(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_adds_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> SaturatedAdd(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_adds_epu16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> SaturatedAdd(const Vec128<int8_t, N> a,
const Vec128<int8_t, N> b) {
return Vec128<int8_t, N>{_mm_adds_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> SaturatedAdd(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_adds_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> SaturatedSub(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_subs_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> SaturatedSub(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_subs_epu16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> SaturatedSub(const Vec128<int8_t, N> a,
const Vec128<int8_t, N> b) {
return Vec128<int8_t, N>{_mm_subs_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> SaturatedSub(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_subs_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> AverageRound(const Vec128<uint8_t, N> a,
const Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_avg_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> AverageRound(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_avg_epu16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> operator*(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_mullo_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> operator*(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_mullo_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> MulHigh(const Vec128<uint16_t, N> a,
const Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_mulhi_epu16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> MulHigh(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_mulhi_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, (N + 1) / 2> MulEven(const Vec128<uint32_t, N> a,
const Vec128<uint32_t, N> b) {
return Vec128<uint64_t, (N + 1) / 2>{_mm_mul_epu32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, (N + 1) / 2> MulEven(const Vec128<int32_t, N> a,
const Vec128<int32_t, N> b) {
return Vec128<int64_t, (N + 1) / 2>{_mm_mul_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> operator*(const Vec128<uint32_t, N> a,
const Vec128<uint32_t, N> b) {
return Vec128<uint32_t, N>{_mm_mullo_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> operator*(const Vec128<int32_t, N> a,
const Vec128<int32_t, N> b) {
const DFromV<decltype(a)> d;
const RebindToUnsigned<decltype(d)> du;
return BitCast(d, BitCast(du, a) * BitCast(du, b));
}
template <int kBits, typename T, size_t N,
hwy::EnableIf<((size_t{1} << sizeof(T)) & ((1 << 1) | (1 << 2))) != 0>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> RotateRight(const Vec128<T, N> v) {
constexpr size_t kSizeInBits = sizeof(T) * 8;
static_assert(0 <= kBits && kBits < kSizeInBits, "Invalid shift count");
if (kBits == 0) return v;
return Or(ShiftRight<kBits>(v),
ShiftLeft<((kSizeInBits - 1) < (kSizeInBits - kBits) ? (kSizeInBits - 1) : (kSizeInBits - kBits))>(v));
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> RotateRight(const Vec128<uint32_t, N> v) {
static_assert(0 <= kBits && kBits < 32, "Invalid shift count");
if (kBits == 0) return v;
return Or(ShiftRight<kBits>(v), ShiftLeft<((31) < (32 - kBits) ? (31) : (32 - kBits))>(v));
}
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> RotateRight(const Vec128<uint64_t, N> v) {
static_assert(0 <= kBits && kBits < 64, "Invalid shift count");
if (kBits == 0) return v;
return Or(ShiftRight<kBits>(v), ShiftLeft<((63) < (64 - kBits) ? (63) : (64 - kBits))>(v));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> BroadcastSignBit(const Vec128<int8_t, N> v) {
const DFromV<decltype(v)> d;
return VecFromMask(v < Zero(d));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> BroadcastSignBit(const Vec128<int16_t, N> v) {
return ShiftRight<15>(v);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> BroadcastSignBit(const Vec128<int32_t, N> v) {
return ShiftRight<31>(v);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> BroadcastSignBit(const Vec128<int64_t, N> v) {
const DFromV<decltype(v)> d;
return VecFromMask(v < Zero(d));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> Abs(const Vec128<int8_t, N> v) {
return Vec128<int8_t, N>{_mm_abs_epi8(v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> Abs(const Vec128<int16_t, N> v) {
return Vec128<int16_t, N>{_mm_abs_epi16(v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> Abs(const Vec128<int32_t, N> v) {
return Vec128<int32_t, N>{_mm_abs_epi32(v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> Abs(const Vec128<int64_t, N> v) {
const auto zero = Zero(DFromV<decltype(v)>());
return IfThenElse(MaskFromVec(BroadcastSignBit(v)), zero - v, v);
}
using Shift64Count = int;
template <int kBits, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> ShiftRight(const Vec128<int64_t, N> v) {
const DFromV<decltype(v)> di;
const RebindToUnsigned<decltype(di)> du;
const auto right = BitCast(di, ShiftRight<kBits>(BitCast(du, v)));
const auto sign = ShiftLeft<64 - kBits>(BroadcastSignBit(v));
return right | sign;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ZeroIfNegative(Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only works for float");
const DFromV<decltype(v)> d;
const auto mask = MaskFromVec(v);
return IfThenElse(mask, Zero(d), v);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> IfNegativeThenElse(const Vec128<int8_t, N> v,
const Vec128<int8_t, N> yes,
const Vec128<int8_t, N> no) {
return IfThenElse(MaskFromVec(v), yes, no);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfNegativeThenElse(Vec128<T, N> v, Vec128<T, N> yes,
Vec128<T, N> no) {
static_assert(IsSigned<T>(), "Only works for signed/float");
const DFromV<decltype(v)> d;
const RebindToSigned<decltype(d)> di;
v = BitCast(d, BroadcastSignBit(BitCast(di, v)));
return IfThenElse(MaskFromVec(v), yes, no);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) != (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfNegativeThenElse(Vec128<T, N> v, Vec128<T, N> yes,
Vec128<T, N> no) {
static_assert(IsSigned<T>(), "Only works for signed/float");
const DFromV<decltype(v)> d;
const RebindToFloat<decltype(d)> df;
return BitCast(d, IfThenElse(MaskFromVec(BitCast(df, v)), BitCast(df, yes),
BitCast(df, no)));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> ShiftLeftSame(const Vec128<uint16_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint16_t, N>{_mm_slli_epi16(v.raw, bits)};
}
return Vec128<uint16_t, N>{_mm_sll_epi16(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> ShiftLeftSame(const Vec128<uint32_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint32_t, N>{_mm_slli_epi32(v.raw, bits)};
}
return Vec128<uint32_t, N>{_mm_sll_epi32(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> ShiftLeftSame(const Vec128<uint64_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint64_t, N>{_mm_slli_epi64(v.raw, bits)};
}
return Vec128<uint64_t, N>{_mm_sll_epi64(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> ShiftLeftSame(const Vec128<int16_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<int16_t, N>{_mm_slli_epi16(v.raw, bits)};
}
return Vec128<int16_t, N>{_mm_sll_epi16(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> ShiftLeftSame(const Vec128<int32_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<int32_t, N>{_mm_slli_epi32(v.raw, bits)};
}
return Vec128<int32_t, N>{_mm_sll_epi32(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> ShiftLeftSame(const Vec128<int64_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<int64_t, N>{_mm_slli_epi64(v.raw, bits)};
}
return Vec128<int64_t, N>{_mm_sll_epi64(v.raw, _mm_cvtsi32_si128(bits))};
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftLeftSame(const Vec128<T, N> v, const int bits) {
const DFromV<decltype(v)> d8;
const Vec128<T, N> shifted{
ShiftLeftSame(Vec128<MakeWide<T>>{v.raw}, bits).raw};
return shifted & Set(d8, static_cast<T>((0xFF << bits) & 0xFF));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> ShiftRightSame(const Vec128<uint16_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint16_t, N>{_mm_srli_epi16(v.raw, bits)};
}
return Vec128<uint16_t, N>{_mm_srl_epi16(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> ShiftRightSame(const Vec128<uint32_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint32_t, N>{_mm_srli_epi32(v.raw, bits)};
}
return Vec128<uint32_t, N>{_mm_srl_epi32(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> ShiftRightSame(const Vec128<uint64_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<uint64_t, N>{_mm_srli_epi64(v.raw, bits)};
}
return Vec128<uint64_t, N>{_mm_srl_epi64(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> ShiftRightSame(Vec128<uint8_t, N> v,
const int bits) {
const DFromV<decltype(v)> d8;
const Vec128<uint8_t, N> shifted{
ShiftRightSame(Vec128<uint16_t>{v.raw}, bits).raw};
return shifted & Set(d8, static_cast<uint8_t>(0xFF >> bits));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> ShiftRightSame(const Vec128<int16_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<int16_t, N>{_mm_srai_epi16(v.raw, bits)};
}
return Vec128<int16_t, N>{_mm_sra_epi16(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> ShiftRightSame(const Vec128<int32_t, N> v,
const int bits) {
if (__builtin_constant_p(bits)) {
return Vec128<int32_t, N>{_mm_srai_epi32(v.raw, bits)};
}
return Vec128<int32_t, N>{_mm_sra_epi32(v.raw, _mm_cvtsi32_si128(bits))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> ShiftRightSame(const Vec128<int64_t, N> v,
const int bits) {
const DFromV<decltype(v)> di;
const RebindToUnsigned<decltype(di)> du;
const auto right = BitCast(di, ShiftRightSame(BitCast(du, v), bits));
const auto sign = ShiftLeftSame(BroadcastSignBit(v), 64 - bits);
return right | sign;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> ShiftRightSame(Vec128<int8_t, N> v, const int bits) {
const DFromV<decltype(v)> di;
const RebindToUnsigned<decltype(di)> du;
const auto shifted = BitCast(di, ShiftRightSame(BitCast(du, v), bits));
const auto shifted_sign =
BitCast(di, Set(du, static_cast<uint8_t>(0x80 >> bits)));
return (shifted ^ shifted_sign) - shifted_sign;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> operator*(Vec128<float, N> a, Vec128<float, N> b) {
return Vec128<float, N>{_mm_mul_ps(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, 1> operator*(const Vec128<float, 1> a,
const Vec128<float, 1> b) {
return Vec128<float, 1>{_mm_mul_ss(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> operator*(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_mul_pd(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<double> operator*(const Vec64<double> a, const Vec64<double> b) {
return Vec64<double>{_mm_mul_sd(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> operator/(const Vec128<float, N> a,
const Vec128<float, N> b) {
return Vec128<float, N>{_mm_div_ps(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, 1> operator/(const Vec128<float, 1> a,
const Vec128<float, 1> b) {
return Vec128<float, 1>{_mm_div_ss(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> operator/(const Vec128<double, N> a,
const Vec128<double, N> b) {
return Vec128<double, N>{_mm_div_pd(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<double> operator/(const Vec64<double> a, const Vec64<double> b) {
return Vec64<double>{_mm_div_sd(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> ApproximateReciprocal(const Vec128<float, N> v) {
return Vec128<float, N>{_mm_rcp_ps(v.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, 1> ApproximateReciprocal(const Vec128<float, 1> v) {
return Vec128<float, 1>{_mm_rcp_ss(v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> AbsDiff(Vec128<float, N> a, Vec128<float, N> b) {
return Abs(a - b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> MulAdd(Vec128<float, N> mul, Vec128<float, N> x,
Vec128<float, N> add) {
return mul * x + add;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> MulAdd(Vec128<double, N> mul, Vec128<double, N> x,
Vec128<double, N> add) {
return mul * x + add;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> NegMulAdd(Vec128<float, N> mul, Vec128<float, N> x,
Vec128<float, N> add) {
return add - mul * x;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> NegMulAdd(Vec128<double, N> mul, Vec128<double, N> x,
Vec128<double, N> add) {
return add - mul * x;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> MulSub(Vec128<float, N> mul, Vec128<float, N> x,
Vec128<float, N> sub) {
return mul * x - sub;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> MulSub(Vec128<double, N> mul, Vec128<double, N> x,
Vec128<double, N> sub) {
return mul * x - sub;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> NegMulSub(Vec128<float, N> mul, Vec128<float, N> x,
Vec128<float, N> sub) {
return Neg(mul) * x - sub;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> NegMulSub(Vec128<double, N> mul, Vec128<double, N> x,
Vec128<double, N> sub) {
return Neg(mul) * x - sub;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Sqrt(Vec128<float, N> v) {
return Vec128<float, N>{_mm_sqrt_ps(v.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, 1> Sqrt(Vec128<float, 1> v) {
return Vec128<float, 1>{_mm_sqrt_ss(v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Sqrt(Vec128<double, N> v) {
return Vec128<double, N>{_mm_sqrt_pd(v.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<double> Sqrt(Vec64<double> v) {
return Vec64<double>{_mm_sqrt_sd(_mm_setzero_pd(), v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> ApproximateReciprocalSqrt(Vec128<float, N> v) {
return Vec128<float, N>{_mm_rsqrt_ps(v.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, 1> ApproximateReciprocalSqrt(Vec128<float, 1> v) {
return Vec128<float, 1>{_mm_rsqrt_ss(v.raw)};
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) __attribute__((unused)) Vec128<T, N> MinU(const Vec128<T, N> a,
const Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const RebindToUnsigned<decltype(d)> du;
const RebindToSigned<decltype(d)> di;
const auto msb = Set(du, static_cast<T>(T(1) << (sizeof(T) * 8 - 1)));
const auto gt = RebindMask(du, BitCast(di, a ^ msb) > BitCast(di, b ^ msb));
return IfThenElse(gt, b, a);
}
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> Min(Vec128<uint8_t, N> a, Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_min_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> Min(Vec128<uint16_t, N> a, Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_min_epu16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> Min(Vec128<uint32_t, N> a, Vec128<uint32_t, N> b) {
return Vec128<uint32_t, N>{_mm_min_epu32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> Min(Vec128<uint64_t, N> a, Vec128<uint64_t, N> b) {
return detail::MinU(a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> Min(Vec128<int8_t, N> a, Vec128<int8_t, N> b) {
return Vec128<int8_t, N>{_mm_min_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> Min(Vec128<int16_t, N> a, Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_min_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> Min(Vec128<int32_t, N> a, Vec128<int32_t, N> b) {
return Vec128<int32_t, N>{_mm_min_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> Min(Vec128<int64_t, N> a, Vec128<int64_t, N> b) {
return IfThenElse(a < b, a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Min(Vec128<float, N> a, Vec128<float, N> b) {
return Vec128<float, N>{_mm_min_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Min(Vec128<double, N> a, Vec128<double, N> b) {
return Vec128<double, N>{_mm_min_pd(a.raw, b.raw)};
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) __attribute__((unused)) Vec128<T, N> MaxU(const Vec128<T, N> a,
const Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const RebindToUnsigned<decltype(d)> du;
const RebindToSigned<decltype(d)> di;
const auto msb = Set(du, static_cast<T>(T(1) << (sizeof(T) * 8 - 1)));
const auto gt = RebindMask(du, BitCast(di, a ^ msb) > BitCast(di, b ^ msb));
return IfThenElse(gt, a, b);
}
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> Max(Vec128<uint8_t, N> a, Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_max_epu8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> Max(Vec128<uint16_t, N> a, Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_max_epu16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> Max(Vec128<uint32_t, N> a, Vec128<uint32_t, N> b) {
return Vec128<uint32_t, N>{_mm_max_epu32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> Max(Vec128<uint64_t, N> a, Vec128<uint64_t, N> b) {
return detail::MaxU(a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> Max(Vec128<int8_t, N> a, Vec128<int8_t, N> b) {
return Vec128<int8_t, N>{_mm_max_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> Max(Vec128<int16_t, N> a, Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_max_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> Max(Vec128<int32_t, N> a, Vec128<int32_t, N> b) {
return Vec128<int32_t, N>{_mm_max_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> Max(Vec128<int64_t, N> a, Vec128<int64_t, N> b) {
return IfThenElse(a < b, b, a);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Max(Vec128<float, N> a, Vec128<float, N> b) {
return Vec128<float, N>{_mm_max_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Max(Vec128<double, N> a, Vec128<double, N> b) {
return Vec128<double, N>{_mm_max_pd(a.raw, b.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<!hwy::IsFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Stream(VFromD<D> v, D , TFromD<D>* __restrict__ aligned) {
_mm_stream_si128(reinterpret_cast<__m128i*>(aligned), v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Stream(VFromD<D> v, D , float* __restrict__ aligned) {
_mm_stream_ps(aligned, v.raw);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Stream(VFromD<D> v, D , double* __restrict__ aligned) {
_mm_stream_pd(aligned, v.raw);
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-conversion"
using GatherIndex64 = long long int;
static_assert(sizeof(GatherIndex64) == 8, "Must be 64-bit type");
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>, class VI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void ScatterOffset(VFromD<D> v, D d, T* __restrict__ base, VI offset) {
using TI = TFromV<VI>;
static_assert(sizeof(T) == sizeof(TI), "Index/lane size must match");
alignas(16) T lanes[MaxLanes(d)];
Store(v, d, lanes);
alignas(16) TI offset_lanes[MaxLanes(d)];
Store(offset, Rebind<TI, decltype(d)>(), offset_lanes);
uint8_t* base_bytes = reinterpret_cast<uint8_t*>(base);
for (size_t i = 0; i < MaxLanes(d); ++i) {
CopyBytes<sizeof(T)>(&lanes[i], base_bytes + offset_lanes[i]);
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>, class VI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void ScatterIndex(VFromD<D> v, D d, T* __restrict__ base, VI index) {
using TI = TFromV<VI>;
static_assert(sizeof(T) == sizeof(TI), "Index/lane size must match");
alignas(16) T lanes[MaxLanes(d)];
Store(v, d, lanes);
alignas(16) TI index_lanes[MaxLanes(d)];
Store(index, Rebind<TI, decltype(d)>(), index_lanes);
for (size_t i = 0; i < MaxLanes(d); ++i) {
base[index_lanes[i]] = lanes[i];
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>, class VI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> GatherOffset(D d, const T* __restrict__ base, VI offset) {
using TI = TFromV<VI>;
static_assert(sizeof(T) == sizeof(TI), "Index/lane size must match");
alignas(16) TI offset_lanes[MaxLanes(d)];
Store(offset, Rebind<TI, decltype(d)>(), offset_lanes);
alignas(16) T lanes[MaxLanes(d)];
const uint8_t* base_bytes = reinterpret_cast<const uint8_t*>(base);
for (size_t i = 0; i < MaxLanes(d); ++i) {
CopyBytes<sizeof(T)>(base_bytes + offset_lanes[i], &lanes[i]);
}
return Load(d, lanes);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename T = TFromD<D>, class VI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> GatherIndex(D d, const T* __restrict__ base, VI index) {
using TI = TFromV<VI>;
static_assert(sizeof(T) == sizeof(TI), "Index/lane size must match");
alignas(16) TI index_lanes[MaxLanes(d)];
Store(index, Rebind<TI, decltype(d)>(), index_lanes);
alignas(16) T lanes[MaxLanes(d)];
for (size_t i = 0; i < MaxLanes(d); ++i) {
lanes[i] = base[index_lanes[i]];
}
return Load(d, lanes);
}
#pragma GCC diagnostic pop
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> LowerHalf(D , VFromD<Twice<D>> v) {
return VFromD<D>{v.raw};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N / 2> LowerHalf(Vec128<T, N> v) {
return Vec128<T, N / 2>{v.raw};
}
template <int kBytes, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ShiftLeftBytes(D d, VFromD<D> v) {
static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes");
const RebindToUnsigned<decltype(d)> du;
return BitCast(
d, VFromD<decltype(du)>{_mm_slli_si128(BitCast(du, v).raw, kBytes)});
}
template <int kBytes, typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftLeftBytes(const Vec128<T, N> v) {
return ShiftLeftBytes<kBytes>(DFromV<decltype(v)>(), v);
}
template <int kLanes, class D, typename T = TFromD<D>,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ShiftLeftLanes(D d, const VFromD<D> v) {
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, ShiftLeftBytes<kLanes * sizeof(T)>(BitCast(d8, v)));
}
template <int kLanes, typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftLeftLanes(const Vec128<T, N> v) {
return ShiftLeftLanes<kLanes>(DFromV<decltype(v)>(), v);
}
template <int kBytes, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ShiftRightBytes(D d, VFromD<D> v) {
static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes");
const RebindToUnsigned<decltype(d)> du;
if (d.MaxBytes() != 16) {
const Full128<TFromD<D>> dfull;
const VFromD<decltype(dfull)> vfull{v.raw};
v = VFromD<D>{IfThenElseZero(FirstN(dfull, MaxLanes(d)), vfull).raw};
}
return BitCast(
d, VFromD<decltype(du)>{_mm_srli_si128(BitCast(du, v).raw, kBytes)});
}
template <int kLanes, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ShiftRightLanes(D d, const VFromD<D> v) {
const Repartition<uint8_t, decltype(d)> d8;
constexpr size_t kBytes = kLanes * sizeof(TFromD<D>);
return BitCast(d, ShiftRightBytes<kBytes>(d8, BitCast(d8, v)));
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> UpperHalf(D , Vec128<T> v) {
return Vec64<T>{_mm_unpackhi_epi64(v.raw, v.raw)};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<float> UpperHalf(D , Vec128<float> v) {
return Vec64<float>{_mm_movehl_ps(v.raw, v.raw)};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<double> UpperHalf(D , Vec128<double> v) {
return Vec64<double>{_mm_unpackhi_pd(v.raw, v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> UpperHalf(D d, VFromD<Twice<D>> v) {
return LowerHalf(d, ShiftRightBytes<d.MaxBytes()>(Twice<D>(), v));
}
namespace detail {
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
inline __attribute__((always_inline)) T ExtractLane(const Vec128<T, N> v) {
static_assert(kLane < N, "Lane index out of bounds");
return static_cast<T>(_mm_extract_epi8(v.raw, kLane) & 0xFF);
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
inline __attribute__((always_inline)) T ExtractLane(const Vec128<T, N> v) {
static_assert(kLane < N, "Lane index out of bounds");
return static_cast<T>(_mm_extract_epi16(v.raw, kLane) & 0xFFFF);
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) T ExtractLane(const Vec128<T, N> v) {
static_assert(kLane < N, "Lane index out of bounds");
return static_cast<T>(_mm_extract_epi32(v.raw, kLane));
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) T ExtractLane(const Vec128<T, N> v) {
static_assert(kLane < N, "Lane index out of bounds");
alignas(16) T lanes[2];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[kLane];
}
template <size_t kLane, size_t N>
inline __attribute__((always_inline)) float ExtractLane(const Vec128<float, N> v) {
static_assert(kLane < N, "Lane index out of bounds");
const int32_t bits = _mm_extract_ps(v.raw, kLane);
float ret;
CopySameSize(&bits, &ret);
return ret;
}
template <size_t kLane>
inline __attribute__((always_inline)) double ExtractLane(const Vec128<double, 1> v) {
static_assert(kLane == 0, "Lane index out of bounds");
return GetLane(v);
}
template <size_t kLane>
inline __attribute__((always_inline)) double ExtractLane(const Vec128<double> v) {
static_assert(kLane < 2, "Lane index out of bounds");
const Half<DFromV<decltype(v)>> dh;
return kLane == 0 ? GetLane(v) : GetLane(UpperHalf(dh, v));
}
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T ExtractLane(const Vec128<T, 1> v, size_t i) {
do { } while (0);
(void)i;
return GetLane(v);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T ExtractLane(const Vec128<T, 2> v, size_t i) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::ExtractLane<0>(v);
case 1:
return detail::ExtractLane<1>(v);
}
}
alignas(16) T lanes[2];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[i];
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T ExtractLane(const Vec128<T, 4> v, size_t i) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::ExtractLane<0>(v);
case 1:
return detail::ExtractLane<1>(v);
case 2:
return detail::ExtractLane<2>(v);
case 3:
return detail::ExtractLane<3>(v);
}
}
alignas(16) T lanes[4];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[i];
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T ExtractLane(const Vec128<T, 8> v, size_t i) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::ExtractLane<0>(v);
case 1:
return detail::ExtractLane<1>(v);
case 2:
return detail::ExtractLane<2>(v);
case 3:
return detail::ExtractLane<3>(v);
case 4:
return detail::ExtractLane<4>(v);
case 5:
return detail::ExtractLane<5>(v);
case 6:
return detail::ExtractLane<6>(v);
case 7:
return detail::ExtractLane<7>(v);
}
}
alignas(16) T lanes[8];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[i];
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T ExtractLane(const Vec128<T, 16> v, size_t i) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::ExtractLane<0>(v);
case 1:
return detail::ExtractLane<1>(v);
case 2:
return detail::ExtractLane<2>(v);
case 3:
return detail::ExtractLane<3>(v);
case 4:
return detail::ExtractLane<4>(v);
case 5:
return detail::ExtractLane<5>(v);
case 6:
return detail::ExtractLane<6>(v);
case 7:
return detail::ExtractLane<7>(v);
case 8:
return detail::ExtractLane<8>(v);
case 9:
return detail::ExtractLane<9>(v);
case 10:
return detail::ExtractLane<10>(v);
case 11:
return detail::ExtractLane<11>(v);
case 12:
return detail::ExtractLane<12>(v);
case 13:
return detail::ExtractLane<13>(v);
case 14:
return detail::ExtractLane<14>(v);
case 15:
return detail::ExtractLane<15>(v);
}
}
alignas(16) T lanes[16];
Store(v, DFromV<decltype(v)>(), lanes);
return lanes[i];
}
namespace detail {
template <class V>
inline __attribute__((always_inline)) V InsertLaneUsingBroadcastAndBlend(V v, size_t i, TFromV<V> t) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
const auto mask = RebindMask(d, Iota(du, 0) == Set(du, static_cast<TU>(i)));
return IfThenElse(mask, Set(d, t), v);
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> InsertLane(const Vec128<T, N> v, T t) {
static_assert(kLane < N, "Lane index out of bounds");
return Vec128<T, N>{_mm_insert_epi8(v.raw, t, kLane)};
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> InsertLane(const Vec128<T, N> v, T t) {
static_assert(kLane < N, "Lane index out of bounds");
return Vec128<T, N>{_mm_insert_epi16(v.raw, t, kLane)};
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> InsertLane(const Vec128<T, N> v, T t) {
static_assert(kLane < N, "Lane index out of bounds");
MakeSigned<T> ti;
CopySameSize(&t, &ti);
return Vec128<T, N>{_mm_insert_epi32(v.raw, ti, kLane)};
}
template <size_t kLane, typename T, size_t N, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> InsertLane(const Vec128<T, N> v, T t) {
static_assert(kLane < N, "Lane index out of bounds");
const DFromV<decltype(v)> d;
const RebindToFloat<decltype(d)> df;
const auto vt = BitCast(df, Set(d, t));
if (kLane == 0) {
return BitCast(
d, Vec128<double, N>{_mm_shuffle_pd(vt.raw, BitCast(df, v).raw, 2)});
}
return BitCast(
d, Vec128<double, N>{_mm_shuffle_pd(BitCast(df, v).raw, vt.raw, 0)});
}
template <size_t kLane, size_t N>
inline __attribute__((always_inline)) Vec128<float, N> InsertLane(const Vec128<float, N> v, float t) {
static_assert(kLane < N, "Lane index out of bounds");
return Vec128<float, N>{_mm_insert_ps(v.raw, _mm_set_ss(t), kLane << 4)};
}
template <size_t kLane>
inline __attribute__((always_inline)) Vec128<double, 1> InsertLane(const Vec128<double, 1> v, double t) {
static_assert(kLane == 0, "Lane index out of bounds");
return Set(DFromV<decltype(v)>(), t);
}
template <size_t kLane>
inline __attribute__((always_inline)) Vec128<double> InsertLane(const Vec128<double> v, double t) {
static_assert(kLane < 2, "Lane index out of bounds");
const DFromV<decltype(v)> d;
const Vec128<double> vt = Set(d, t);
if (kLane == 0) {
return Vec128<double>{_mm_shuffle_pd(vt.raw, v.raw, 2)};
}
return Vec128<double>{_mm_shuffle_pd(v.raw, vt.raw, 0)};
}
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> InsertLane(const Vec128<T, 1> v, size_t i, T t) {
do { } while (0);
(void)i;
return Set(DFromV<decltype(v)>(), t);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 2> InsertLane(const Vec128<T, 2> v, size_t i, T t) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::InsertLane<0>(v, t);
case 1:
return detail::InsertLane<1>(v, t);
}
}
return detail::InsertLaneUsingBroadcastAndBlend(v, i, t);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 4> InsertLane(const Vec128<T, 4> v, size_t i, T t) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::InsertLane<0>(v, t);
case 1:
return detail::InsertLane<1>(v, t);
case 2:
return detail::InsertLane<2>(v, t);
case 3:
return detail::InsertLane<3>(v, t);
}
}
return detail::InsertLaneUsingBroadcastAndBlend(v, i, t);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 8> InsertLane(const Vec128<T, 8> v, size_t i, T t) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::InsertLane<0>(v, t);
case 1:
return detail::InsertLane<1>(v, t);
case 2:
return detail::InsertLane<2>(v, t);
case 3:
return detail::InsertLane<3>(v, t);
case 4:
return detail::InsertLane<4>(v, t);
case 5:
return detail::InsertLane<5>(v, t);
case 6:
return detail::InsertLane<6>(v, t);
case 7:
return detail::InsertLane<7>(v, t);
}
}
return detail::InsertLaneUsingBroadcastAndBlend(v, i, t);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 16> InsertLane(const Vec128<T, 16> v, size_t i, T t) {
if (__builtin_constant_p(i)) {
switch (i) {
case 0:
return detail::InsertLane<0>(v, t);
case 1:
return detail::InsertLane<1>(v, t);
case 2:
return detail::InsertLane<2>(v, t);
case 3:
return detail::InsertLane<3>(v, t);
case 4:
return detail::InsertLane<4>(v, t);
case 5:
return detail::InsertLane<5>(v, t);
case 6:
return detail::InsertLane<6>(v, t);
case 7:
return detail::InsertLane<7>(v, t);
case 8:
return detail::InsertLane<8>(v, t);
case 9:
return detail::InsertLane<9>(v, t);
case 10:
return detail::InsertLane<10>(v, t);
case 11:
return detail::InsertLane<11>(v, t);
case 12:
return detail::InsertLane<12>(v, t);
case 13:
return detail::InsertLane<13>(v, t);
case 14:
return detail::InsertLane<14>(v, t);
case 15:
return detail::InsertLane<15>(v, t);
}
}
return detail::InsertLaneUsingBroadcastAndBlend(v, i, t);
}
template <int kBytes, class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> CombineShiftRightBytes(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, Vec128<uint8_t>{_mm_alignr_epi8(
BitCast(d8, hi).raw, BitCast(d8, lo).raw, kBytes)});
}
template <int kBytes, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> CombineShiftRightBytes(D d, VFromD<D> hi, VFromD<D> lo) {
constexpr size_t kSize = d.MaxBytes();
static_assert(0 < kBytes && kBytes < kSize, "kBytes invalid");
const Repartition<uint8_t, decltype(d)> d8;
using V8 = Vec128<uint8_t>;
const DFromV<V8> dfull8;
const Repartition<TFromD<D>, decltype(dfull8)> dfull;
const V8 hi8{BitCast(d8, hi).raw};
const V8 lo8 = ShiftLeftBytes<16 - kSize>(V8{BitCast(d8, lo).raw});
const V8 r = CombineShiftRightBytes<16 - kSize + kBytes>(dfull8, hi8, lo8);
return VFromD<D>{BitCast(dfull, r).raw};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> Broadcast(const Vec128<uint16_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
if (kLane < 4) {
const __m128i lo = _mm_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF);
return Vec128<uint16_t, N>{_mm_unpacklo_epi64(lo, lo)};
} else {
const __m128i hi = _mm_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF);
return Vec128<uint16_t, N>{_mm_unpackhi_epi64(hi, hi)};
}
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> Broadcast(const Vec128<uint32_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<uint32_t, N>{_mm_shuffle_epi32(v.raw, 0x55 * kLane)};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> Broadcast(const Vec128<uint64_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<uint64_t, N>{_mm_shuffle_epi32(v.raw, kLane ? 0xEE : 0x44)};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> Broadcast(const Vec128<int16_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
if (kLane < 4) {
const __m128i lo = _mm_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF);
return Vec128<int16_t, N>{_mm_unpacklo_epi64(lo, lo)};
} else {
const __m128i hi = _mm_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF);
return Vec128<int16_t, N>{_mm_unpackhi_epi64(hi, hi)};
}
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> Broadcast(const Vec128<int32_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<int32_t, N>{_mm_shuffle_epi32(v.raw, 0x55 * kLane)};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> Broadcast(const Vec128<int64_t, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<int64_t, N>{_mm_shuffle_epi32(v.raw, kLane ? 0xEE : 0x44)};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Broadcast(const Vec128<float, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<float, N>{_mm_shuffle_ps(v.raw, v.raw, 0x55 * kLane)};
}
template <int kLane, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Broadcast(const Vec128<double, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
return Vec128<double, N>{_mm_shuffle_pd(v.raw, v.raw, 3 * kLane)};
}
template <typename T, size_t N = 16 / sizeof(T)>
struct Indices128 {
__m128i raw;
};
template <class D, typename T = TFromD<D>, typename TI, size_t kN,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Indices128<T, kN> IndicesFromVec(D d, Vec128<TI, kN> vec) {
static_assert(sizeof(T) == sizeof(TI), "Index size must match lane");
(void)d;
return Indices128<T, kN>{vec.raw};
}
template <class D, typename T = TFromD<D>, typename TI, size_t kN,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Indices128<T, kN> IndicesFromVec(D d, Vec128<TI, kN> vec) {
static_assert(sizeof(T) == sizeof(TI), "Index size must match lane");
const Repartition<uint8_t, decltype(d)> d8;
using V8 = VFromD<decltype(d8)>;
alignas(16) static constexpr uint8_t kByteOffsets[16] = {
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1};
alignas(16) static constexpr uint8_t kBroadcastLaneBytes[16] = {
0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14};
const V8 lane_indices = TableLookupBytes(vec, Load(d8, kBroadcastLaneBytes));
const Repartition<uint16_t, decltype(d)> d16;
const V8 byte_indices = BitCast(d8, ShiftLeft<1>(BitCast(d16, lane_indices)));
return Indices128<T, kN>{Add(byte_indices, Load(d8, kByteOffsets)).raw};
}
template <class D, typename T = TFromD<D>, typename TI, size_t kN,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Indices128<T, kN> IndicesFromVec(D d, Vec128<TI, kN> vec) {
static_assert(sizeof(T) == sizeof(TI), "Index size must match lane");
const Repartition<uint8_t, decltype(d)> d8;
using V8 = VFromD<decltype(d8)>;
alignas(16) static constexpr uint8_t kByteOffsets[16] = {
0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3};
alignas(16) static constexpr uint8_t kBroadcastLaneBytes[16] = {
0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12};
const V8 lane_indices = TableLookupBytes(vec, Load(d8, kBroadcastLaneBytes));
const Repartition<uint16_t, decltype(d)> d16;
const V8 byte_indices = BitCast(d8, ShiftLeft<2>(BitCast(d16, lane_indices)));
return Indices128<T, kN>{Add(byte_indices, Load(d8, kByteOffsets)).raw};
}
template <class D, typename T = TFromD<D>, typename TI, size_t kN,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Indices128<T, kN> IndicesFromVec(D d, Vec128<TI, kN> vec) {
static_assert(sizeof(T) == sizeof(TI), "Index size must match lane");
(void)d;
return Indices128<T, kN>{vec.raw};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, typename TI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Indices128<TFromD<D>, D::kPrivateLanes> SetTableIndices(
D d, const TI* idx) {
static_assert(sizeof(TFromD<D>) == sizeof(TI), "Index size must match lane");
const Rebind<TI, decltype(d)> di;
return IndicesFromVec(d, LoadU(di, idx));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> TableLookupLanes(Vec128<T, N> v, Indices128<T, N> idx) {
return TableLookupBytes(v, Vec128<T, N>{idx.raw});
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> TableLookupLanes(Vec128<T, N> v, Indices128<T, N> idx) {
return TableLookupBytes(v, Vec128<T, N>{idx.raw});
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> TableLookupLanes(Vec128<T, N> v, Indices128<T, N> idx) {
return TableLookupBytes(v, Vec128<T, N>{idx.raw});
}
template <size_t N, hwy::EnableIf<(N * sizeof(float) > 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> TableLookupLanes(Vec128<float, N> v,
Indices128<float, N> idx) {
const DFromV<decltype(v)> df;
const RebindToSigned<decltype(df)> di;
return BitCast(df,
TableLookupBytes(BitCast(di, v), Vec128<int32_t, N>{idx.raw}));
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> TableLookupLanes(Vec128<T, 1> v,
Indices128<T, 1> ) {
return v;
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> TableLookupLanes(Vec128<T> v, Indices128<T> idx) {
const DFromV<decltype(v)> d;
Vec128<int64_t> vidx{idx.raw};
const RebindToSigned<decltype(d)> di;
const Vec128<int64_t> same = (vidx ^ Iota(di, 0)) - Set(di, 1);
const Mask128<T> mask_same = RebindMask(d, MaskFromVec(same));
return IfThenElse(mask_same, v, Shuffle01(v));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> TableLookupLanes(Vec128<double> v,
Indices128<double> idx) {
Vec128<int64_t> vidx{idx.raw};
const DFromV<decltype(v)> d;
const RebindToSigned<decltype(d)> di;
const Vec128<int64_t> same = (vidx ^ Iota(di, 0)) - Set(di, 1);
const Mask128<double> mask_same = RebindMask(d, MaskFromVec(same));
return IfThenElse(mask_same, v, Shuffle01(v));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReverseBlocks(D , VFromD<D> v) {
return v;
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> Reverse(D , Vec128<T, 1> v) {
return v;
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> Reverse(D , const Vec64<T> v) {
return Vec64<T>{Shuffle2301(Vec128<T>{v.raw}).raw};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Reverse(D , const Vec128<T> v) {
return Shuffle01(v);
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Reverse(D , const Vec128<T> v) {
return Shuffle0123(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse(D d, const VFromD<D> v) {
constexpr size_t kN = MaxLanes(d);
if (kN == 1) return v;
if (kN == 2) {
return VFromD<D>{_mm_shufflelo_epi16(v.raw,
(((
0
) << 6) | ((
1
) << 4) | ((
0
) << 2) | (
1
))
)};
}
if (kN == 4) {
return VFromD<D>{_mm_shufflelo_epi16(v.raw,
(((
0
) << 6) | ((
1
) << 4) | ((
2
) << 2) | (
3
))
)};
}
const RebindToSigned<decltype(d)> di;
alignas(16) static constexpr int16_t kShuffle[8] = {
0x0F0E, 0x0D0C, 0x0B0A, 0x0908, 0x0706, 0x0504, 0x0302, 0x0100};
return BitCast(d, TableLookupBytes(v, LoadDup128(di, kShuffle)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse(D d, const VFromD<D> v) {
constexpr int kN = static_cast<int>(MaxLanes(d));
if (kN == 1) return v;
alignas(16) static constexpr int8_t kReverse[16] = {
kN - 1, kN - 2, kN - 3, kN - 4, kN - 5, kN - 6, kN - 7, kN - 8,
kN - 9, kN - 10, kN - 11, kN - 12, kN - 13, kN - 14, kN - 15, kN - 16};
const RebindToSigned<decltype(d)> di;
const VFromD<decltype(di)> idx = Load(di, kReverse);
return VFromD<D>{_mm_shuffle_epi8(BitCast(di, v).raw, idx.raw)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> Reverse2(D , Vec128<T, 1> v) {
return v;
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse2(D d, VFromD<D> v) {
const RebindToSigned<decltype(d)> di;
alignas(16) static constexpr int16_t kShuffle[8] = {
0x0302, 0x0100, 0x0706, 0x0504, 0x0B0A, 0x0908, 0x0F0E, 0x0D0C};
return BitCast(d, TableLookupBytes(v, LoadDup128(di, kShuffle)));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse2(D , VFromD<D> v) {
return Shuffle2301(v);
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse2(D , VFromD<D> v) {
return Shuffle01(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse4(D d, VFromD<D> v) {
constexpr size_t kN = MaxLanes(d);
if (kN <= 4) {
return VFromD<D>{_mm_shufflelo_epi16(v.raw,
(((
0
) << 6) | ((
1
) << 4) | ((
2
) << 2) | (
3
))
)};
}
const RebindToSigned<decltype(d)> di;
alignas(16) static constexpr int16_t kShuffle[8] = {
0x0706, 0x0504, 0x0302, 0x0100, 0x0F0E, 0x0D0C, 0x0B0A, 0x0908};
return BitCast(d, TableLookupBytes(v, LoadDup128(di, kShuffle)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse4(D , const VFromD<D> v) {
return Shuffle0123(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse4(D , VFromD<D> ) {
do { if (!(0)) { ::hwy::Abort("/home/mathieu/Perso/highway/hwy/ops/x86_128-inl.h", 4907, "Assert %s","0"); } } while (0);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse8(D d, const VFromD<D> v) {
const RebindToSigned<decltype(d)> di;
alignas(16) static constexpr int16_t kShuffle[8] = {
0x0F0E, 0x0D0C, 0x0B0A, 0x0908, 0x0706, 0x0504, 0x0302, 0x0100};
return BitCast(d, TableLookupBytes(v, LoadDup128(di, kShuffle)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr,
hwy::EnableIf<((size_t{1} << sizeof(TFromD<D>)) & ((1 << 4) | (1 << 8))) != 0>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse8(D , VFromD<D> ) {
do { if (!(0)) { ::hwy::Abort("/home/mathieu/Perso/highway/hwy/ops/x86_128-inl.h", 4928, "Assert %s","0"); } } while (0);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> InterleaveLower(Vec128<uint8_t, N> a,
Vec128<uint8_t, N> b) {
return Vec128<uint8_t, N>{_mm_unpacklo_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> InterleaveLower(Vec128<uint16_t, N> a,
Vec128<uint16_t, N> b) {
return Vec128<uint16_t, N>{_mm_unpacklo_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> InterleaveLower(Vec128<uint32_t, N> a,
Vec128<uint32_t, N> b) {
return Vec128<uint32_t, N>{_mm_unpacklo_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> InterleaveLower(Vec128<uint64_t, N> a,
Vec128<uint64_t, N> b) {
return Vec128<uint64_t, N>{_mm_unpacklo_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> InterleaveLower(Vec128<int8_t, N> a,
Vec128<int8_t, N> b) {
return Vec128<int8_t, N>{_mm_unpacklo_epi8(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> InterleaveLower(Vec128<int16_t, N> a,
Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_unpacklo_epi16(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> InterleaveLower(Vec128<int32_t, N> a,
Vec128<int32_t, N> b) {
return Vec128<int32_t, N>{_mm_unpacklo_epi32(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> InterleaveLower(Vec128<int64_t, N> a,
Vec128<int64_t, N> b) {
return Vec128<int64_t, N>{_mm_unpacklo_epi64(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> InterleaveLower(Vec128<float, N> a,
Vec128<float, N> b) {
return Vec128<float, N>{_mm_unpacklo_ps(a.raw, b.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> InterleaveLower(Vec128<double, N> a,
Vec128<double, N> b) {
return Vec128<double, N>{_mm_unpacklo_pd(a.raw, b.raw)};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> InterleaveLower(D , VFromD<D> a, VFromD<D> b) {
return InterleaveLower(a, b);
}
namespace detail {
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t> InterleaveUpper(Vec128<uint8_t> a, Vec128<uint8_t> b) {
return Vec128<uint8_t>{_mm_unpackhi_epi8(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t> InterleaveUpper(Vec128<uint16_t> a,
Vec128<uint16_t> b) {
return Vec128<uint16_t>{_mm_unpackhi_epi16(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> InterleaveUpper(Vec128<uint32_t> a,
Vec128<uint32_t> b) {
return Vec128<uint32_t>{_mm_unpackhi_epi32(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t> InterleaveUpper(Vec128<uint64_t> a,
Vec128<uint64_t> b) {
return Vec128<uint64_t>{_mm_unpackhi_epi64(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t> InterleaveUpper(Vec128<int8_t> a, Vec128<int8_t> b) {
return Vec128<int8_t>{_mm_unpackhi_epi8(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t> InterleaveUpper(Vec128<int16_t> a, Vec128<int16_t> b) {
return Vec128<int16_t>{_mm_unpackhi_epi16(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> InterleaveUpper(Vec128<int32_t> a, Vec128<int32_t> b) {
return Vec128<int32_t>{_mm_unpackhi_epi32(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t> InterleaveUpper(Vec128<int64_t> a, Vec128<int64_t> b) {
return Vec128<int64_t>{_mm_unpackhi_epi64(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> InterleaveUpper(Vec128<float> a, Vec128<float> b) {
return Vec128<float>{_mm_unpackhi_ps(a.raw, b.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> InterleaveUpper(Vec128<double> a, Vec128<double> b) {
return Vec128<double>{_mm_unpackhi_pd(a.raw, b.raw)};
}
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> InterleaveUpper(D , Vec128<T> a, Vec128<T> b) {
return detail::InterleaveUpper(a, b);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> InterleaveUpper(D d, VFromD<D> a, VFromD<D> b) {
const Half<decltype(d)> d2;
return InterleaveLower(d, VFromD<D>{UpperHalf(d2, a).raw},
VFromD<D>{UpperHalf(d2, b).raw});
}
template <int kLane, class T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Broadcast(const Vec128<T, N> v) {
static_assert(0 <= kLane && kLane < N, "Invalid lane");
const DFromV<decltype(v)> d;
return TableLookupBytes(v, Set(d, static_cast<T>(kLane)));
}
template <class V, class DW = RepartitionToWide<DFromV<V>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DW> ZipLower(V a, V b) {
return BitCast(DW(), InterleaveLower(a, b));
}
template <class V, class D = DFromV<V>, class DW = RepartitionToWide<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DW> ZipLower(DW dw, V a, V b) {
return BitCast(dw, InterleaveLower(D(), a, b));
}
template <class V, class D = DFromV<V>, class DW = RepartitionToWide<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DW> ZipUpper(DW dw, V a, V b) {
return BitCast(dw, InterleaveUpper(D(), a, b));
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> Per4LaneBlkShufDupSet4xU32(D d, const uint32_t x3,
const uint32_t x2,
const uint32_t x1,
const uint32_t x0) {
return ResizeBitCast(
d, Vec128<uint32_t>{_mm_set_epi32(
static_cast<int32_t>(x3), static_cast<int32_t>(x2),
static_cast<int32_t>(x1), static_cast<int32_t>(x0))});
}
template <size_t kIdx3210, class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> ,
hwy::SizeTag<2> ,
hwy::SizeTag<8> , V v) {
return V{_mm_shufflelo_epi16(v.raw, static_cast<int>(kIdx3210 & 0xFF))};
}
template <size_t kIdx3210, class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >()>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> ,
hwy::SizeTag<4> ,
hwy::SizeTag<16> , V v) {
return V{_mm_shuffle_epi32(v.raw, static_cast<int>(kIdx3210 & 0xFF))};
}
template <size_t kIdx3210, class V, hwy::EnableIf<hwy::IsFloat<TFromV<V> >()>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> ,
hwy::SizeTag<4> ,
hwy::SizeTag<16> , V v) {
return V{_mm_shuffle_ps(v.raw, v.raw, static_cast<int>(kIdx3210 & 0xFF))};
}
}
namespace detail {
template <class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 8>* = nullptr>
inline __attribute__((always_inline)) V SlideUpLanes(V v, size_t amt) {
const DFromV<decltype(v)> d;
const Full64<uint64_t> du64;
const auto vu64 = ResizeBitCast(du64, v);
return ResizeBitCast(
d, ShiftLeftSame(vu64, static_cast<int>(amt * sizeof(TFromV<V>) * 8)));
}
template <class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) == 16>* = nullptr>
inline __attribute__((always_inline)) V SlideUpLanes(V v, size_t amt) {
const DFromV<decltype(v)> d;
const Repartition<uint8_t, decltype(d)> du8;
const auto idx =
Iota(du8, static_cast<uint8_t>(size_t{0} - amt * sizeof(TFromV<V>)));
return BitCast(d, TableLookupBytesOr0(BitCast(du8, v), idx));
}
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpLanes(D , VFromD<D> v, size_t ) {
return v;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftLeftLanes<1>(d, v);
}
}
return detail::SlideUpLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftLeftLanes<1>(d, v);
case 2:
return ShiftLeftLanes<2>(d, v);
case 3:
return ShiftLeftLanes<3>(d, v);
}
}
return detail::SlideUpLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftLeftLanes<1>(d, v);
case 2:
return ShiftLeftLanes<2>(d, v);
case 3:
return ShiftLeftLanes<3>(d, v);
case 4:
return ShiftLeftLanes<4>(d, v);
case 5:
return ShiftLeftLanes<5>(d, v);
case 6:
return ShiftLeftLanes<6>(d, v);
case 7:
return ShiftLeftLanes<7>(d, v);
}
}
return detail::SlideUpLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 16)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftLeftLanes<1>(d, v);
case 2:
return ShiftLeftLanes<2>(d, v);
case 3:
return ShiftLeftLanes<3>(d, v);
case 4:
return ShiftLeftLanes<4>(d, v);
case 5:
return ShiftLeftLanes<5>(d, v);
case 6:
return ShiftLeftLanes<6>(d, v);
case 7:
return ShiftLeftLanes<7>(d, v);
case 8:
return ShiftLeftLanes<8>(d, v);
case 9:
return ShiftLeftLanes<9>(d, v);
case 10:
return ShiftLeftLanes<10>(d, v);
case 11:
return ShiftLeftLanes<11>(d, v);
case 12:
return ShiftLeftLanes<12>(d, v);
case 13:
return ShiftLeftLanes<13>(d, v);
case 14:
return ShiftLeftLanes<14>(d, v);
case 15:
return ShiftLeftLanes<15>(d, v);
}
}
return detail::SlideUpLanes(v, amt);
}
namespace detail {
template <class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 8>* = nullptr>
inline __attribute__((always_inline)) V SlideDownLanes(V v, size_t amt) {
const DFromV<decltype(v)> d;
const Repartition<UnsignedFromSize<d.MaxBytes()>, decltype(d)> dv;
return BitCast(d,
ShiftRightSame(BitCast(dv, v),
static_cast<int>(amt * sizeof(TFromV<V>) * 8)));
}
template <class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) == 16>* = nullptr>
inline __attribute__((always_inline)) V SlideDownLanes(V v, size_t amt) {
const DFromV<decltype(v)> d;
const Repartition<int8_t, decltype(d)> di8;
auto idx = Iota(di8, static_cast<int8_t>(amt * sizeof(TFromV<V>)));
idx = Or(idx, VecFromMask(di8, idx > Set(di8, int8_t{15})));
return BitCast(d, TableLookupBytesOr0(BitCast(di8, v), idx));
}
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownLanes(D , VFromD<D> v, size_t ) {
return v;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftRightLanes<1>(d, v);
}
}
return detail::SlideDownLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftRightLanes<1>(d, v);
case 2:
return ShiftRightLanes<2>(d, v);
case 3:
return ShiftRightLanes<3>(d, v);
}
}
return detail::SlideDownLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftRightLanes<1>(d, v);
case 2:
return ShiftRightLanes<2>(d, v);
case 3:
return ShiftRightLanes<3>(d, v);
case 4:
return ShiftRightLanes<4>(d, v);
case 5:
return ShiftRightLanes<5>(d, v);
case 6:
return ShiftRightLanes<6>(d, v);
case 7:
return ShiftRightLanes<7>(d, v);
}
}
return detail::SlideDownLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 16)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownLanes(D d, VFromD<D> v, size_t amt) {
if (__builtin_constant_p(amt)) {
switch (amt) {
case 0:
return v;
case 1:
return ShiftRightLanes<1>(d, v);
case 2:
return ShiftRightLanes<2>(d, v);
case 3:
return ShiftRightLanes<3>(d, v);
case 4:
return ShiftRightLanes<4>(d, v);
case 5:
return ShiftRightLanes<5>(d, v);
case 6:
return ShiftRightLanes<6>(d, v);
case 7:
return ShiftRightLanes<7>(d, v);
case 8:
return ShiftRightLanes<8>(d, v);
case 9:
return ShiftRightLanes<9>(d, v);
case 10:
return ShiftRightLanes<10>(d, v);
case 11:
return ShiftRightLanes<11>(d, v);
case 12:
return ShiftRightLanes<12>(d, v);
case 13:
return ShiftRightLanes<13>(d, v);
case 14:
return ShiftRightLanes<14>(d, v);
case 15:
return ShiftRightLanes<15>(d, v);
}
}
return detail::SlideDownLanes(v, amt);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, class VH = VFromD<Half<D>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Combine(D d, VH hi_half, VH lo_half) {
const Half<decltype(d)> dh;
const RebindToUnsigned<decltype(dh)> duh;
using VU = Vec128<UnsignedFromSize<dh.MaxBytes()>, 2>;
const VU lo{BitCast(duh, lo_half).raw};
const VU hi{BitCast(duh, hi_half).raw};
return BitCast(d, InterleaveLower(lo, hi));
}
namespace detail {
template <class D, typename T = TFromD<D>>
inline __attribute__((always_inline)) Vec128<T> ZeroExtendVector(hwy::NonFloatTag , D ,
Vec64<T> lo) {
return Vec128<T>{_mm_move_epi64(lo.raw)};
}
template <class D, typename T = TFromD<D>>
inline __attribute__((always_inline)) Vec128<T> ZeroExtendVector(hwy::FloatTag , D d, Vec64<T> lo) {
const RebindToUnsigned<decltype(d)> du;
return BitCast(d, ZeroExtendVector(du, BitCast(Half<decltype(du)>(), lo)));
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ZeroExtendVector(D d, Vec64<T> lo) {
return detail::ZeroExtendVector(hwy::IsFloatTag<T>(), d, lo);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ZeroExtendVector(D d, VFromD<Half<D>> lo) {
const Half<D> dh;
return IfThenElseZero(FirstN(d, MaxLanes(dh)), VFromD<D>{lo.raw});
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatLowerLower(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<uint64_t, decltype(d)> d64;
return BitCast(d, InterleaveLower(BitCast(d64, lo), BitCast(d64, hi)));
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatUpperUpper(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<uint64_t, decltype(d)> d64;
return BitCast(d, InterleaveUpper(d64, BitCast(d64, lo), BitCast(d64, hi)));
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatLowerUpper(D d, Vec128<T> hi, Vec128<T> lo) {
return CombineShiftRightBytes<8>(d, hi, lo);
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatUpperLower(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<double, decltype(d)> dd;
return BitCast(d, Vec128<double>{_mm_blend_pd(BitCast(dd, hi).raw,
BitCast(dd, lo).raw, 1)});
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> ConcatUpperLower(D d, Vec128<float> hi,
Vec128<float> lo) {
const RepartitionToWide<decltype(d)> dd;
return BitCast(d, Vec128<double>{_mm_blend_pd(BitCast(dd, hi).raw,
BitCast(dd, lo).raw, 1)});
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> ConcatUpperLower(D , Vec128<double> hi,
Vec128<double> lo) {
return Vec128<double>{_mm_blend_pd(hi.raw, lo.raw, 1)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatLowerLower(D d, VFromD<D> hi, VFromD<D> lo) {
const Half<decltype(d)> d2;
return Combine(d, LowerHalf(d2, hi), LowerHalf(d2, lo));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatUpperUpper(D d, VFromD<D> hi, VFromD<D> lo) {
const Half<decltype(d)> d2;
return Combine(d, UpperHalf(d2, hi), UpperHalf(d2, lo));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatLowerUpper(D d, const VFromD<D> hi,
const VFromD<D> lo) {
const Half<decltype(d)> d2;
return Combine(d, LowerHalf(d2, hi), UpperHalf(d2, lo));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatUpperLower(D d, VFromD<D> hi, VFromD<D> lo) {
const Half<decltype(d)> d2;
return Combine(d, UpperHalf(d2, hi), LowerHalf(d2, lo));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatOdd(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<uint16_t, decltype(d)> dw;
const Vec128<uint16_t> uH = ShiftRight<8>(BitCast(dw, hi));
const Vec128<uint16_t> uL = ShiftRight<8>(BitCast(dw, lo));
return Vec128<T>{_mm_packus_epi16(uL.raw, uH.raw)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ConcatOdd(D d, Vec64<T> hi, Vec64<T> lo) {
const Repartition<uint32_t, decltype(d)> du32;
alignas(16) const uint8_t kCompactOddU8[8] = {1, 3, 5, 7};
const Vec64<T> shuf = BitCast(d, Load(Full64<uint8_t>(), kCompactOddU8));
const Vec64<T> L = TableLookupBytes(lo, shuf);
const Vec64<T> H = TableLookupBytes(hi, shuf);
return BitCast(d, InterleaveLower(du32, BitCast(du32, L), BitCast(du32, H)));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<T> ConcatOdd(D d, Vec32<T> hi, Vec32<T> lo) {
const Repartition<uint16_t, decltype(d)> du16;
alignas(16) const uint8_t kCompactOddU8[4] = {1, 3};
const Vec32<T> shuf = BitCast(d, Load(Full32<uint8_t>(), kCompactOddU8));
const Vec32<T> L = TableLookupBytes(lo, shuf);
const Vec32<T> H = TableLookupBytes(hi, shuf);
return BitCast(d, InterleaveLower(du16, BitCast(du16, L), BitCast(du16, H)));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatOdd(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<int32_t, decltype(d)> dw;
const Vec128<int32_t> uH = ShiftRight<16>(BitCast(dw, hi));
const Vec128<int32_t> uL = ShiftRight<16>(BitCast(dw, lo));
return Vec128<T>{_mm_packs_epi32(uL.raw, uH.raw)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ConcatOdd(D d, Vec64<T> hi, Vec64<T> lo) {
const Repartition<uint32_t, decltype(d)> du32;
alignas(16) const uint8_t kCompactOddU16[8] = {2, 3, 6, 7};
const Vec64<T> shuf = BitCast(d, Load(Full64<uint8_t>(), kCompactOddU16));
const Vec64<T> L = TableLookupBytes(lo, shuf);
const Vec64<T> H = TableLookupBytes(hi, shuf);
return BitCast(d, InterleaveLower(du32, BitCast(du32, L), BitCast(du32, H)));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatOdd(D d, Vec128<T> hi, Vec128<T> lo) {
const RebindToFloat<decltype(d)> df;
return BitCast(
d, Vec128<float>{_mm_shuffle_ps(BitCast(df, lo).raw, BitCast(df, hi).raw,
(((
3
) << 6) | ((
1
) << 4) | ((
3
) << 2) | (
1
))
)});
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> ConcatOdd(D , Vec128<float> hi, Vec128<float> lo) {
return Vec128<float>{_mm_shuffle_ps(lo.raw, hi.raw,
(((
3
) << 6) | ((
1
) << 4) | ((
3
) << 2) | (
1
))
)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 2> ConcatOdd(D d, Vec128<T, 2> hi, Vec128<T, 2> lo) {
return InterleaveUpper(d, lo, hi);
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatEven(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<uint16_t, decltype(d)> dw;
const Vec128<uint16_t> mask = Set(dw, 0x00FF);
const Vec128<uint16_t> uH = And(BitCast(dw, hi), mask);
const Vec128<uint16_t> uL = And(BitCast(dw, lo), mask);
return Vec128<T>{_mm_packus_epi16(uL.raw, uH.raw)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ConcatEven(D d, Vec64<T> hi, Vec64<T> lo) {
const Repartition<uint32_t, decltype(d)> du32;
alignas(16) const uint8_t kCompactEvenU8[8] = {0, 2, 4, 6};
const Vec64<T> shuf = BitCast(d, Load(Full64<uint8_t>(), kCompactEvenU8));
const Vec64<T> L = TableLookupBytes(lo, shuf);
const Vec64<T> H = TableLookupBytes(hi, shuf);
return BitCast(d, InterleaveLower(du32, BitCast(du32, L), BitCast(du32, H)));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<T> ConcatEven(D d, Vec32<T> hi, Vec32<T> lo) {
const Repartition<uint16_t, decltype(d)> du16;
alignas(16) const uint8_t kCompactEvenU8[4] = {0, 2};
const Vec32<T> shuf = BitCast(d, Load(Full32<uint8_t>(), kCompactEvenU8));
const Vec32<T> L = TableLookupBytes(lo, shuf);
const Vec32<T> H = TableLookupBytes(hi, shuf);
return BitCast(d, InterleaveLower(du16, BitCast(du16, L), BitCast(du16, H)));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatEven(D d, Vec128<T> hi, Vec128<T> lo) {
const Repartition<uint32_t, decltype(d)> dw;
const Vec128<uint32_t> mask = Set(dw, 0x0000FFFF);
const Vec128<uint32_t> uH = And(BitCast(dw, hi), mask);
const Vec128<uint32_t> uL = And(BitCast(dw, lo), mask);
return Vec128<T>{_mm_packus_epi32(uL.raw, uH.raw)};
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> ConcatEven(D d, Vec64<T> hi, Vec64<T> lo) {
const Repartition<uint32_t, decltype(d)> du32;
alignas(16) const uint8_t kCompactEvenU16[8] = {0, 1, 4, 5};
const Vec64<T> shuf = BitCast(d, Load(Full64<uint8_t>(), kCompactEvenU16));
const Vec64<T> L = TableLookupBytes(lo, shuf);
const Vec64<T> H = TableLookupBytes(hi, shuf);
return BitCast(d, InterleaveLower(du32, BitCast(du32, L), BitCast(du32, H)));
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> ConcatEven(D d, Vec128<T> hi, Vec128<T> lo) {
const RebindToFloat<decltype(d)> df;
return BitCast(
d, Vec128<float>{_mm_shuffle_ps(BitCast(df, lo).raw, BitCast(df, hi).raw,
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)});
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> ConcatEven(D , Vec128<float> hi,
Vec128<float> lo) {
return Vec128<float>{_mm_shuffle_ps(lo.raw, hi.raw,
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)};
}
template <typename D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 2> ConcatEven(D d, Vec128<T, 2> hi, Vec128<T, 2> lo) {
return InterleaveLower(d, lo, hi);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> DupEven(const Vec128<T, 1> v) {
return v;
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 2> DupEven(const Vec128<T, 2> v) {
return InterleaveLower(DFromV<decltype(v)>(), v, v);
}
template <typename V, hwy::EnableIf<sizeof(TFromV<V>) == (1)>* = nullptr, hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) > 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V DupEven(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint8_t kShuffle[16] = {
0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14};
return TableLookupBytes(v, BitCast(d, LoadDup128(du, kShuffle)));
}
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<T> DupEven(const Vec64<T> v) {
return Vec64<T>{_mm_shufflelo_epi16(v.raw,
(((
2
) << 6) | ((
2
) << 4) | ((
0
) << 2) | (
0
))
)};
}
template <typename V, hwy::EnableIf<sizeof(TFromV<V>) == (2)>* = nullptr, hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V DupEven(V v) {
alignas(16) static constexpr uint16_t kShuffle[8] = {
0x0100, 0x0100, 0x0504, 0x0504, 0x0908, 0x0908, 0x0d0c, 0x0d0c};
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
return TableLookupBytes(v, BitCast(d, LoadDup128(du, kShuffle)));
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> DupEven(Vec128<T> v) {
return Vec128<T>{_mm_shuffle_epi32(v.raw,
(((
2
) << 6) | ((
2
) << 4) | ((
0
) << 2) | (
0
))
)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> DupEven(Vec128<float> v) {
return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw,
(((
2
) << 6) | ((
2
) << 4) | ((
0
) << 2) | (
0
))
)};
}
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> DupOdd(Vec128<T, 1> v) {
return v;
}
template <typename V, hwy::EnableIf<sizeof(TFromV<V>) == (1)>* = nullptr, hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V DupOdd(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint8_t kShuffle[16] = {
1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15};
return TableLookupBytes(v, BitCast(d, LoadDup128(du, kShuffle)));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr, hwy::EnableIf<(N <= 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> DupOdd(Vec128<T, N> v) {
return Vec128<T, N>{_mm_shufflelo_epi16(v.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
}
template <typename V, hwy::EnableIf<sizeof(TFromV<V>) == (2)>* = nullptr, hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V DupOdd(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint16_t kShuffle[8] = {
0x0302, 0x0302, 0x0706, 0x0706, 0x0b0a, 0x0b0a, 0x0f0e, 0x0f0e};
return TableLookupBytes(v, BitCast(d, LoadDup128(du, kShuffle)));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> DupOdd(Vec128<T, N> v) {
return Vec128<T, N>{_mm_shuffle_epi32(v.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> DupOdd(Vec128<float, N> v) {
return Vec128<float, N>{
_mm_shuffle_ps(v.raw, v.raw,
(((
3
) << 6) | ((
3
) << 4) | ((
1
) << 2) | (
1
))
)};
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> DupOdd(const Vec128<T, N> v) {
return InterleaveUpper(DFromV<decltype(v)>(), v, v);
}
template <typename T, size_t N, hwy::EnableIf<N * sizeof(T) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> TwoTablesLookupLanes(Vec128<T, N> a, Vec128<T, N> b,
Indices128<T, N> idx) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
const Indices128<T, N * 2> idx2{idx.raw};
return LowerHalf(d, TableLookupLanes(Combine(dt, b, a), idx2));
}
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> TwoTablesLookupLanes(Vec128<T> a, Vec128<T> b,
Indices128<T> idx) {
const DFromV<decltype(a)> d;
const Vec128<T> idx_vec{idx.raw};
const Repartition<uint16_t, decltype(d)> du16;
const auto sel_hi_mask =
MaskFromVec(BitCast(d, ShiftLeft<3>(BitCast(du16, idx_vec))));
const auto lo_lookup_result = TableLookupBytes(a, idx_vec);
const auto hi_lookup_result = TableLookupBytes(b, idx_vec);
return IfThenElse(sel_hi_mask, hi_lookup_result, lo_lookup_result);
}
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> TwoTablesLookupLanes(Vec128<T> a, Vec128<T> b,
Indices128<T> idx) {
const DFromV<decltype(a)> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, TwoTablesLookupLanes(BitCast(du8, a), BitCast(du8, b),
Indices128<uint8_t>{idx.raw}));
}
template <typename T, hwy::EnableIf<IsSame<T, uint32_t>() || IsSame<T, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> TwoTablesLookupLanes(Vec128<T> a, Vec128<T> b,
Indices128<T> idx) {
const DFromV<decltype(a)> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, TwoTablesLookupLanes(BitCast(du8, a), BitCast(du8, b),
Indices128<uint8_t>{idx.raw}));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float> TwoTablesLookupLanes(Vec128<float> a, Vec128<float> b,
Indices128<float> idx) {
const DFromV<decltype(a)> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, TwoTablesLookupLanes(BitCast(du8, a), BitCast(du8, b),
Indices128<uint8_t>{idx.raw}));
}
template <typename T, hwy::EnableIf<IsSame<T, uint64_t>() || IsSame<T, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> TwoTablesLookupLanes(Vec128<T> a, Vec128<T> b,
Indices128<T> idx) {
const DFromV<decltype(a)> d;
const Vec128<T> idx_vec{idx.raw};
const Indices128<T> idx_mod{And(idx_vec, Set(d, T{1})).raw};
const RebindToFloat<decltype(d)> d_sel;
const auto sel_hi_mask = MaskFromVec(BitCast(d_sel, ShiftLeft<62>(idx_vec)));
const auto lo_lookup_result = BitCast(d_sel, TableLookupLanes(a, idx_mod));
const auto hi_lookup_result = BitCast(d_sel, TableLookupLanes(b, idx_mod));
return BitCast(d,
IfThenElse(sel_hi_mask, hi_lookup_result, lo_lookup_result));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double> TwoTablesLookupLanes(Vec128<double> a, Vec128<double> b,
Indices128<double> idx) {
const DFromV<decltype(a)> d;
const RebindToSigned<decltype(d)> di;
const Vec128<int64_t> idx_vec{idx.raw};
const Indices128<double> idx_mod{And(idx_vec, Set(di, int64_t{1})).raw};
const auto sel_hi_mask = MaskFromVec(BitCast(d, ShiftLeft<62>(idx_vec)));
const auto lo_lookup_result = TableLookupLanes(a, idx_mod);
const auto hi_lookup_result = TableLookupLanes(b, idx_mod);
return IfThenElse(sel_hi_mask, hi_lookup_result, lo_lookup_result);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> OddEven(const Vec128<T, N> a, const Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const Repartition<uint8_t, decltype(d)> d8;
alignas(16) static constexpr uint8_t mask[16] = {
0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0};
return IfThenElse(MaskFromVec(BitCast(d, Load(d8, mask))), b, a);
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> OddEven(const Vec128<T, N> a, const Vec128<T, N> b) {
return Vec128<T, N>{_mm_blend_epi16(a.raw, b.raw, 0x55)};
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> OddEven(const Vec128<T, N> a, const Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const RebindToFloat<decltype(d)> df;
return BitCast(d, Vec128<float, N>{_mm_blend_ps(BitCast(df, a).raw,
BitCast(df, b).raw, 5)});
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, N> OddEven(const Vec128<T, N> a, const Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const RebindToFloat<decltype(d)> dd;
return BitCast(d, Vec128<double, N>{_mm_blend_pd(BitCast(dd, a).raw,
BitCast(dd, b).raw, 1)});
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> OddEven(Vec128<float, N> a, Vec128<float, N> b) {
return Vec128<float, N>{_mm_blend_ps(a.raw, b.raw, 5)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> OddEvenBlocks(Vec128<T, N> , Vec128<T, N> even) {
return even;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> SwapAdjacentBlocks(Vec128<T, N> v) {
return v;
}
namespace detail {
template <typename T, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<MakeUnsigned<T>> Pow2(const Vec128<T> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const RepartitionToWide<decltype(d)> dw;
const Rebind<float, decltype(dw)> df;
const auto zero = Zero(d);
const auto exp = ShiftLeft<23 - 16>(v);
const auto upper = exp + Set(d, 0x3F80);
const auto f0 = ZipLower(dw, zero, upper);
const auto f1 = ZipUpper(dw, zero, upper);
const VFromD<decltype(dw)> bits0{_mm_cvtps_epi32(BitCast(df, f0).raw)};
const VFromD<decltype(dw)> bits1{_mm_cvtps_epi32(BitCast(df, f1).raw)};
return VFromD<decltype(du)>{_mm_packus_epi32(bits0.raw, bits1.raw)};
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr, hwy::EnableIf<(N <= 4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<MakeUnsigned<T>, N> Pow2(const Vec128<T, N> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const Twice<decltype(du)> dt_u;
const RepartitionToWide<decltype(dt_u)> dt_w;
const RebindToFloat<decltype(dt_w)> dt_f;
const auto exp = ShiftLeft<23 - 16>(v);
const auto upper = exp + Set(d, 0x3F80);
const auto f0 = ZipLower(dt_w, Zero(dt_u), ResizeBitCast(dt_u, upper));
const VFromD<decltype(dt_w)> bits0{_mm_cvtps_epi32(BitCast(dt_f, f0).raw)};
return VFromD<decltype(du)>{_mm_packus_epi32(bits0.raw, bits0.raw)};
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<MakeUnsigned<T>, N> Pow2(const Vec128<T, N> v) {
const DFromV<decltype(v)> d;
const auto exp = ShiftLeft<23>(v);
const auto f = exp + Set(d, 0x3F800000);
return Vec128<MakeUnsigned<T>, N>{_mm_cvtps_epi32(_mm_castsi128_ps(f.raw))};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> Shl(hwy::UnsignedTag , Vec128<uint16_t, N> v,
Vec128<uint16_t, N> bits) {
return v * Pow2(bits);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec16<uint16_t> Shl(hwy::UnsignedTag , Vec16<uint16_t> v,
Vec16<uint16_t> bits) {
const Vec16<uint16_t> bits16{_mm_cvtepu16_epi64(bits.raw)};
return Vec16<uint16_t>{_mm_sll_epi16(v.raw, bits16.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> Shl(hwy::UnsignedTag tag, Vec128<uint8_t, N> v,
Vec128<uint8_t, N> bits) {
const DFromV<decltype(v)> d;
const Repartition<uint16_t, decltype(d)> dw;
using VW = VFromD<decltype(dw)>;
const VW even_mask = Set(dw, 0x00FF);
const VW odd_mask = Set(dw, 0xFF00);
const VW vw = BitCast(dw, v);
const VW bits16 = BitCast(dw, bits);
const VW evens = Shl(tag, vw, And(bits16, even_mask));
const VW odds = Shl(tag, And(vw, odd_mask), ShiftRight<8>(bits16));
return OddEven(BitCast(d, odds), BitCast(d, evens));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, 1> Shl(hwy::UnsignedTag , Vec128<uint8_t, 1> v,
Vec128<uint8_t, 1> bits) {
const Vec16<uint16_t> bits8{_mm_cvtepu8_epi64(bits.raw)};
return Vec128<uint8_t, 1>{_mm_sll_epi16(v.raw, bits8.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> Shl(hwy::UnsignedTag , Vec128<uint32_t, N> v,
Vec128<uint32_t, N> bits) {
return v * Pow2(bits);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<uint32_t> Shl(hwy::UnsignedTag , Vec32<uint32_t> v,
const Vec32<uint32_t> bits) {
const Vec32<uint32_t> bits32{_mm_cvtepu32_epi64(bits.raw)};
return Vec32<uint32_t>{_mm_sll_epi32(v.raw, bits32.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t> Shl(hwy::UnsignedTag , Vec128<uint64_t> v,
Vec128<uint64_t> bits) {
const DFromV<decltype(v)> d;
const Vec128<uint64_t> out0{_mm_sll_epi64(v.raw, bits.raw)};
const __m128i bits1 = _mm_unpackhi_epi64(bits.raw, bits.raw);
const Vec128<uint64_t> out1{_mm_sll_epi64(v.raw, bits1)};
return ConcatUpperLower(d, out1, out0);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<uint64_t> Shl(hwy::UnsignedTag , Vec64<uint64_t> v,
Vec64<uint64_t> bits) {
return Vec64<uint64_t>{_mm_sll_epi64(v.raw, bits.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Shl(hwy::SignedTag , Vec128<T, N> v,
Vec128<T, N> bits) {
const DFromV<decltype(v)> di;
const RebindToUnsigned<decltype(di)> du;
return BitCast(di,
Shl(hwy::UnsignedTag(), BitCast(du, v), BitCast(du, bits)));
}
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator<<(Vec128<T, N> v, Vec128<T, N> bits) {
return detail::Shl(hwy::TypeTag<T>(), v, bits);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> operator>>(Vec128<uint16_t, N> in,
const Vec128<uint16_t, N> bits) {
const DFromV<decltype(in)> d;
const auto out = MulHigh(in, detail::Pow2(Set(d, 16) - bits));
return IfThenElse(bits == Zero(d), in, out);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec16<uint16_t> operator>>(const Vec16<uint16_t> in,
const Vec16<uint16_t> bits) {
const Vec16<uint16_t> bits16{_mm_cvtepu16_epi64(bits.raw)};
return Vec16<uint16_t>{_mm_srl_epi16(in.raw, bits16.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> operator>>(Vec128<uint8_t, N> in,
const Vec128<uint8_t, N> bits) {
const DFromV<decltype(in)> d;
const Repartition<uint16_t, decltype(d)> dw;
using VW = VFromD<decltype(dw)>;
const VW mask = Set(dw, 0x00FF);
const VW vw = BitCast(dw, in);
const VW bits16 = BitCast(dw, bits);
const VW evens = And(vw, mask) >> And(bits16, mask);
const VW odds = vw >> ShiftRight<8>(bits16);
return OddEven(BitCast(d, odds), BitCast(d, evens));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, 1> operator>>(const Vec128<uint8_t, 1> in,
const Vec128<uint8_t, 1> bits) {
const Vec16<uint16_t> in8{_mm_cvtepu8_epi16(in.raw)};
const Vec16<uint16_t> bits8{_mm_cvtepu8_epi64(bits.raw)};
return Vec128<uint8_t, 1>{_mm_srl_epi16(in8.raw, bits8.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, N> operator>>(const Vec128<uint32_t, N> in,
const Vec128<uint32_t, N> bits) {
const DFromV<decltype(in)> d32;
const Vec128<uint32_t, N> in31{_mm_shuffle_epi32(in.raw, 0x31)};
const auto mul = detail::Pow2(Set(d32, 32) - bits);
const auto out20 = ShiftRight<32>(MulEven(in, mul));
const Vec128<uint32_t, N> mul31{_mm_shuffle_epi32(mul.raw, 0x31)};
const auto out31 = BitCast(d32, MulEven(in31, mul31));
const Vec128<uint32_t, N> out = OddEven(out31, BitCast(d32, out20));
return IfThenElse(bits == Zero(d32), in, out);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t, 1> operator>>(const Vec128<uint32_t, 1> in,
const Vec128<uint32_t, 1> bits) {
const Vec32<uint32_t> bits32{_mm_cvtepu32_epi64(bits.raw)};
return Vec128<uint32_t, 1>{_mm_srl_epi32(in.raw, bits32.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t> operator>>(const Vec128<uint64_t> v,
const Vec128<uint64_t> bits) {
const DFromV<decltype(v)> d;
const Vec128<uint64_t> out0{_mm_srl_epi64(v.raw, bits.raw)};
const __m128i bits1 = _mm_unpackhi_epi64(bits.raw, bits.raw);
const Vec128<uint64_t> out1{_mm_srl_epi64(v.raw, bits1)};
return ConcatUpperLower(d, out1, out0);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<uint64_t> operator>>(const Vec64<uint64_t> v,
const Vec64<uint64_t> bits) {
return Vec64<uint64_t>{_mm_srl_epi64(v.raw, bits.raw)};
}
namespace detail {
template <class DI, class V>
inline __attribute__((always_inline)) V SignedShr(const DI di, const V v, const V count_i) {
const RebindToUnsigned<DI> du;
const auto count = BitCast(du, count_i);
const auto sign = BroadcastSignBit(v);
const auto abs = BitCast(du, v ^ sign);
return BitCast(di, abs >> count) ^ sign;
}
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> operator>>(Vec128<int16_t, N> v,
Vec128<int16_t, N> bits) {
const DFromV<decltype(v)> d;
return detail::SignedShr(d, v, bits);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec16<int16_t> operator>>(Vec16<int16_t> v, Vec16<int16_t> bits) {
const Vec16<int16_t> bits16{_mm_cvtepu16_epi64(bits.raw)};
return Vec16<int16_t>{_mm_sra_epi16(v.raw, bits16.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, N> operator>>(Vec128<int8_t, N> v,
Vec128<int8_t, N> bits) {
const DFromV<decltype(v)> d;
return detail::SignedShr(d, v, bits);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t, 1> operator>>(Vec128<int8_t, 1> v,
Vec128<int8_t, 1> bits) {
const Vec16<int16_t> vi16{_mm_cvtepi8_epi16(v.raw)};
const Vec16<uint16_t> bits8{_mm_cvtepu8_epi64(bits.raw)};
return Vec128<int8_t, 1>{_mm_sra_epi16(vi16.raw, bits8.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> operator>>(Vec128<int32_t, N> v,
Vec128<int32_t, N> bits) {
const DFromV<decltype(v)> d;
return detail::SignedShr(d, v, bits);
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<int32_t> operator>>(Vec32<int32_t> v, Vec32<int32_t> bits) {
const Vec32<uint32_t> bits32{_mm_cvtepu32_epi64(bits.raw)};
return Vec32<int32_t>{_mm_sra_epi32(v.raw, bits32.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, N> operator>>(Vec128<int64_t, N> v,
Vec128<int64_t, N> bits) {
const DFromV<decltype(v)> d;
return detail::SignedShr(d, v, bits);
}
inline __attribute__((always_inline)) Vec128<uint64_t> MulEven(Vec128<uint64_t> a, Vec128<uint64_t> b) {
const DFromV<decltype(a)> d;
alignas(16) uint64_t mul[2];
mul[0] = Mul128(GetLane(a), GetLane(b), &mul[1]);
return Load(d, mul);
}
inline __attribute__((always_inline)) Vec128<uint64_t> MulOdd(Vec128<uint64_t> a, Vec128<uint64_t> b) {
const DFromV<decltype(a)> d;
const Half<decltype(d)> d2;
alignas(16) uint64_t mul[2];
const uint64_t a1 = GetLane(UpperHalf(d2, a));
const uint64_t b1 = GetLane(UpperHalf(d2, b));
mul[0] = Mul128(a1, b1, &mul[1]);
return Load(d, mul);
}
template <class D32, hwy::EnableIf<IsSame<TFromD<D32>, float>()>* = nullptr,
class V16 = VFromD<Repartition<bfloat16_t, D32>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D32> WidenMulPairwiseAdd(D32 df32, V16 a, V16 b) {
const RebindToUnsigned<decltype(df32)> du32;
using VU32 = VFromD<decltype(du32)>;
const VU32 odd = Set(du32, 0xFFFF0000u);
const VU32 ae = ShiftLeft<16>(BitCast(du32, a));
const VU32 ao = And(BitCast(du32, a), odd);
const VU32 be = ShiftLeft<16>(BitCast(du32, b));
const VU32 bo = And(BitCast(du32, b), odd);
return MulAdd(BitCast(df32, ae), BitCast(df32, be),
Mul(BitCast(df32, ao), BitCast(df32, bo)));
}
template <class D32, hwy::EnableIf<IsSame<TFromD<D32>, int32_t>()>* = nullptr, hwy::EnableIf<D32::kPrivateLanes * sizeof(TFromD<D32>) <= 16>* = nullptr,
class V16 = VFromD<RepartitionToNarrow<D32>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D32> WidenMulPairwiseAdd(D32 , V16 a, V16 b) {
return VFromD<D32>{_mm_madd_epi16(a.raw, b.raw)};
}
template <class D32, hwy::EnableIf<IsSame<TFromD<D32>, float>()>* = nullptr,
class V16 = VFromD<Repartition<bfloat16_t, D32>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D32> ReorderWidenMulAccumulate(D32 df32, V16 a, V16 b,
const VFromD<D32> sum0,
VFromD<D32>& sum1) {
const RebindToUnsigned<decltype(df32)> du32;
using VU32 = VFromD<decltype(du32)>;
const VU32 odd = Set(du32, 0xFFFF0000u);
const VU32 ae = ShiftLeft<16>(BitCast(du32, a));
const VU32 ao = And(BitCast(du32, a), odd);
const VU32 be = ShiftLeft<16>(BitCast(du32, b));
const VU32 bo = And(BitCast(du32, b), odd);
sum1 = MulAdd(BitCast(df32, ao), BitCast(df32, bo), sum1);
return MulAdd(BitCast(df32, ae), BitCast(df32, be), sum0);
}
template <class D32, hwy::EnableIf<IsSame<TFromD<D32>, int32_t>()>* = nullptr, hwy::EnableIf<D32::kPrivateLanes * sizeof(TFromD<D32>) <= 16>* = nullptr,
class V16 = VFromD<RepartitionToNarrow<D32>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D32> ReorderWidenMulAccumulate(D32 d, V16 a, V16 b,
const VFromD<D32> sum0,
VFromD<D32>& ) {
(void)d;
return sum0 + WidenMulPairwiseAdd(d, a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> RearrangeToOddPlusEven(const Vec128<int32_t, N> sum0,
Vec128<int32_t, N> ) {
return sum0;
}
template <class VW>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VW RearrangeToOddPlusEven(const VW sum0, const VW sum1) {
return Add(sum0, sum1);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<uint8_t, D>> v) {
return VFromD<D>{_mm_cvtepu8_epi16(v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<uint16_t, D>> v) {
return VFromD<D>{_mm_cvtepu16_epi32(v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<uint32_t, D>> v) {
return VFromD<D>{_mm_cvtepu32_epi64(v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<uint8_t, D>> v) {
return VFromD<D>{_mm_cvtepu8_epi32(v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D d, VFromD<Rebind<uint8_t, D>> v) {
(void)d;
return VFromD<D>{_mm_cvtepu8_epi64(v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D d, VFromD<Rebind<uint16_t, D>> v) {
(void)d;
return VFromD<D>{_mm_cvtepu16_epi64(v.raw)};
}
template <class D, class V, hwy::EnableIf<IsSigned<TFromD<D> >() && !IsFloat<TFromD<D> >() && !IsSpecialFloat<TFromD<D> >()>* = nullptr, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
hwy::EnableIf<(sizeof(TFromD<D>) > sizeof(TFromV<V>))>* = nullptr,
hwy::EnableIf<(D::kPrivateLanes == DFromV<V>::kPrivateLanes)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D di, V v) {
const RebindToUnsigned<decltype(di)> du;
return BitCast(di, PromoteTo(du, v));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<int8_t, D>> v) {
return VFromD<D>{_mm_cvtepi8_epi16(v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<int16_t, D>> v) {
return VFromD<D>{_mm_cvtepi16_epi32(v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<int32_t, D>> v) {
return VFromD<D>{_mm_cvtepi32_epi64(v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<int8_t, D>> v) {
return VFromD<D>{_mm_cvtepi8_epi32(v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D d, VFromD<Rebind<int8_t, D>> v) {
(void)d;
return VFromD<D>{_mm_cvtepi8_epi64(v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D d, VFromD<Rebind<int16_t, D>> v) {
(void)d;
return VFromD<D>{_mm_cvtepi16_epi64(v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> PromoteTo(D df32, VFromD<Rebind<float16_t, D>> v) {
const RebindToSigned<decltype(df32)> di32;
const RebindToUnsigned<decltype(df32)> du32;
const auto bits16 = PromoteTo(du32, VFromD<Rebind<uint16_t, D>>{v.raw});
const auto sign = ShiftRight<15>(bits16);
const auto biased_exp = ShiftRight<10>(bits16) & Set(du32, 0x1F);
const auto mantissa = bits16 & Set(du32, 0x3FF);
const auto subnormal =
BitCast(du32, ConvertTo(df32, BitCast(di32, mantissa)) *
Set(df32, 1.0f / 16384 / 1024));
const auto biased_exp32 = biased_exp + Set(du32, 127 - 15);
const auto mantissa32 = ShiftLeft<23 - 10>(mantissa);
const auto normal = ShiftLeft<23>(biased_exp32) | mantissa32;
const auto bits32 = IfThenElse(biased_exp == Zero(du32), subnormal, normal);
return BitCast(df32, ShiftLeft<31>(sign) | bits32);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D df32, VFromD<Rebind<bfloat16_t, D>> v) {
const Rebind<uint16_t, decltype(df32)> du16;
const RebindToSigned<decltype(df32)> di32;
return BitCast(df32, ShiftLeft<16>(PromoteTo(di32, BitCast(du16, v))));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<float, D>> v) {
return VFromD<D>{_mm_cvtps_pd(v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , VFromD<Rebind<int32_t, D>> v) {
return VFromD<D>{_mm_cvtepi32_pd(v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int32_t, D>> v) {
return VFromD<D>{_mm_packs_epi32(v.raw, v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int32_t, D>> v) {
return VFromD<D>{_mm_packus_epi32(v.raw, v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D du16, VFromD<Rebind<uint32_t, D>> v) {
const DFromV<decltype(v)> du32;
const RebindToSigned<decltype(du32)> di32;
return DemoteTo(du16, BitCast(di32, Min(v, Set(du32, 0x7FFFFFFF))));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int32_t, D>> v) {
const __m128i i16 = _mm_packs_epi32(v.raw, v.raw);
return VFromD<D>{_mm_packus_epi16(i16, i16)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int16_t, D>> v) {
return VFromD<D>{_mm_packus_epi16(v.raw, v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int32_t, D>> v) {
const __m128i i16 = _mm_packs_epi32(v.raw, v.raw);
return VFromD<D>{_mm_packs_epi16(i16, i16)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<int16_t, D>> v) {
return VFromD<D>{_mm_packs_epi16(v.raw, v.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D du8, VFromD<Rebind<uint32_t, D>> v) {
const DFromV<decltype(v)> du32;
const RebindToSigned<decltype(du32)> di32;
const auto max_i32 = Set(du32, 0x7FFFFFFFu);
const auto clamped = BitCast(di32, Min(v, max_i32));
return DemoteTo(du8, clamped);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D du8, VFromD<Rebind<uint16_t, D>> v) {
const DFromV<decltype(v)> du16;
const RebindToSigned<decltype(du16)> di16;
const auto max_i16 = Set(du16, 0x7FFF);
const auto clamped = BitCast(di16, Min(v, max_i16));
return DemoteTo(du8, clamped);
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmain"
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D df16, VFromD<Rebind<float, D>> v) {
const RebindToUnsigned<decltype(df16)> du16;
const Rebind<uint32_t, decltype(df16)> du;
const RebindToSigned<decltype(du)> di;
const auto bits32 = BitCast(du, v);
const auto sign = ShiftRight<31>(bits32);
const auto biased_exp32 = ShiftRight<23>(bits32) & Set(du, 0xFF);
const auto mantissa32 = bits32 & Set(du, 0x7FFFFF);
const auto k15 = Set(di, 15);
const auto exp = Min(BitCast(di, biased_exp32) - Set(di, 127), k15);
const auto is_tiny = exp < Set(di, -24);
const auto is_subnormal = exp < Set(di, -14);
const auto biased_exp16 =
BitCast(du, IfThenZeroElse(is_subnormal, exp + k15));
const auto sub_exp = BitCast(du, Set(di, -14) - exp);
const auto sub_m = (Set(du, 1) << (Set(du, 10) - sub_exp)) +
(mantissa32 >> (Set(du, 13) + sub_exp));
const auto mantissa16 = IfThenElse(RebindMask(du, is_subnormal), sub_m,
ShiftRight<13>(mantissa32));
const auto sign16 = ShiftLeft<15>(sign);
const auto normal16 = sign16 | ShiftLeft<10>(biased_exp16) | mantissa16;
const auto bits16 = IfThenZeroElse(is_tiny, BitCast(di, normal16));
return BitCast(df16, DemoteTo(du16, bits16));
}
#pragma GCC diagnostic pop
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, bfloat16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D dbf16, VFromD<Rebind<float, D>> v) {
const Rebind<int32_t, decltype(dbf16)> di32;
const Rebind<uint32_t, decltype(dbf16)> du32;
const Rebind<uint16_t, decltype(dbf16)> du16;
const auto bits_in_32 = BitCast(di32, ShiftRight<16>(BitCast(du32, v)));
return BitCast(dbf16, DemoteTo(du16, bits_in_32));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, bfloat16_t>()>* = nullptr,
class V32 = VFromD<Repartition<float, D>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dbf16, V32 a, V32 b) {
const RebindToUnsigned<decltype(dbf16)> du16;
const Repartition<uint32_t, decltype(dbf16)> du32;
const VFromD<decltype(du32)> b_in_even = ShiftRight<16>(BitCast(du32, b));
return BitCast(dbf16, OddEven(BitCast(du16, a), BitCast(du16, b_in_even)));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<int16_t> ReorderDemote2To(D dn, Vec32<int32_t> a,
Vec32<int32_t> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<int16_t> ReorderDemote2To(D , Vec64<int32_t> a,
Vec64<int32_t> b) {
return Vec64<int16_t>{_mm_shuffle_epi32(_mm_packs_epi32(a.raw, b.raw),
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t> ReorderDemote2To(D , Vec128<int32_t> a,
Vec128<int32_t> b) {
return Vec128<int16_t>{_mm_packs_epi32(a.raw, b.raw)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<uint16_t> ReorderDemote2To(D dn, Vec32<int32_t> a,
Vec32<int32_t> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<uint16_t> ReorderDemote2To(D dn, Vec64<int32_t> a,
Vec64<int32_t> b) {
(void)dn;
return Vec64<uint16_t>{_mm_shuffle_epi32(_mm_packus_epi32(a.raw, b.raw),
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t> ReorderDemote2To(D dn, Vec128<int32_t> a,
Vec128<int32_t> b) {
(void)dn;
return Vec128<uint16_t>{_mm_packus_epi32(a.raw, b.raw)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, Vec128<uint32_t> a,
Vec128<uint32_t> b) {
const DFromV<decltype(a)> du32;
const RebindToSigned<decltype(du32)> di32;
const auto max_i32 = Set(du32, 0x7FFFFFFFu);
const auto clamped_a = BitCast(di32, Min(a, max_i32));
const auto clamped_b = BitCast(di32, Min(b, max_i32));
return ReorderDemote2To(dn, clamped_a, clamped_b);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<uint32_t, D>> a,
VFromD<Repartition<uint32_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<int16_t, D>> a,
VFromD<Repartition<int16_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<int8_t> ReorderDemote2To(D , Vec64<int16_t> a,
Vec64<int16_t> b) {
return Vec64<int8_t>{_mm_shuffle_epi32(_mm_packs_epi16(a.raw, b.raw),
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int8_t> ReorderDemote2To(D , Vec128<int16_t> a,
Vec128<int16_t> b) {
return Vec128<int8_t>{_mm_packs_epi16(a.raw, b.raw)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<int16_t, D>> a,
VFromD<Repartition<int16_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<uint8_t> ReorderDemote2To(D , Vec64<int16_t> a,
Vec64<int16_t> b) {
return Vec64<uint8_t>{_mm_shuffle_epi32(_mm_packus_epi16(a.raw, b.raw),
(((
2
) << 6) | ((
0
) << 4) | ((
2
) << 2) | (
0
))
)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t> ReorderDemote2To(D , Vec128<int16_t> a,
Vec128<int16_t> b) {
return Vec128<uint8_t>{_mm_packus_epi16(a.raw, b.raw)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, Vec128<uint16_t> a,
Vec128<uint16_t> b) {
const DFromV<decltype(a)> du16;
const RebindToSigned<decltype(du16)> di16;
const auto max_i16 = Set(du16, 0x7FFFu);
const auto clamped_a = BitCast(di16, Min(a, max_i16));
const auto clamped_b = BitCast(di16, Min(b, max_i16));
return ReorderDemote2To(dn, clamped_a, clamped_b);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<uint16_t, D>> a,
VFromD<Repartition<uint16_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<!hwy::IsFloat<TFromD<D> >() && !hwy::IsSpecialFloat<TFromD<D> >()>* = nullptr,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr,
hwy::EnableIf<sizeof(TFromV<V>) == (sizeof(TFromD<D>) * 2)>* = nullptr,
hwy::EnableIf<(D::kPrivateLanes == DFromV<V>::kPrivateLanes * 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> OrderedDemote2To(D d, V a, V b) {
return ReorderDemote2To(d, a, b);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, bfloat16_t>()>* = nullptr, class V32 = VFromD<Repartition<float, D>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> OrderedDemote2To(D dbf16, V32 a, V32 b) {
const RebindToUnsigned<decltype(dbf16)> du16;
return BitCast(dbf16, ConcatOdd(du16, BitCast(du16, b), BitCast(du16, a)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<Rebind<double, D>> v) {
return VFromD<D>{_mm_cvtpd_ps(v.raw)};
}
namespace detail {
template <class D>
inline __attribute__((always_inline)) VFromD<D> ClampF64ToI32Max(D d, VFromD<D> v) {
return Min(v, Set(d, 2147483647.0));
}
template <class DI, class DF = RebindToFloat<DI>>
inline __attribute__((always_inline)) VFromD<DI> FixConversionOverflow(
DI di, VFromD<DF> original, decltype(Zero(DI()).raw) converted_raw) {
const VFromD<DI> converted{converted_raw};
const VFromD<DI> sign_wrong = AndNot(BitCast(di, original), converted);
const RebindToUnsigned<DI> du;
const VFromD<DI> mask = BroadcastSignBit(sign_wrong);
const VFromD<DI> max = BitCast(di, ShiftRight<1>(BitCast(du, mask)));
return IfVecThenElse(mask, max, converted);
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr,
class DF = Rebind<double, D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , VFromD<DF> v) {
const VFromD<DF> clamped = detail::ClampF64ToI32Max(DF(), v);
return VFromD<D>{_mm_cvttpd_epi32(clamped.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> U8FromU32(const Vec128<uint32_t, N> v) {
const DFromV<decltype(v)> d32;
const Repartition<uint8_t, decltype(d32)> d8;
alignas(16) static constexpr uint32_t k8From32[4] = {
0x0C080400u, 0x0C080400u, 0x0C080400u, 0x0C080400u};
const auto quad = TableLookupBytes(v, Load(d32, k8From32));
return LowerHalf(LowerHalf(BitCast(d8, quad)));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> MulFixedPoint15(const Vec128<int16_t, N> a,
const Vec128<int16_t, N> b) {
return Vec128<int16_t, N>{_mm_mulhrs_epi16(a.raw, b.raw)};
}
template <typename From, class DTo, hwy::EnableIf<(DTo::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DTo> TruncateTo(DTo , Vec128<From, 1> v) {
const Repartition<TFromD<DTo>, DFromV<decltype(v)>> dto;
return VFromD<DTo>{BitCast(dto, v).raw};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec16<uint8_t> TruncateTo(D d, Vec128<uint64_t> v) {
const Repartition<uint8_t, DFromV<decltype(v)>> d8;
(void)d;
alignas(16) static constexpr uint8_t kIdx[16] = {0, 8, 0, 8, 0, 8, 0, 8,
0, 8, 0, 8, 0, 8, 0, 8};
const Vec128<uint8_t> v8 = TableLookupBytes(v, Load(d8, kIdx));
return LowerHalf(LowerHalf(LowerHalf(v8)));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec32<uint16_t> TruncateTo(D d, Vec128<uint64_t> v) {
(void)d;
const Repartition<uint16_t, DFromV<decltype(v)>> d16;
alignas(16) static constexpr uint16_t kIdx[8] = {
0x100u, 0x908u, 0x100u, 0x908u, 0x100u, 0x908u, 0x100u, 0x908u};
const Vec128<uint16_t> v16 = TableLookupBytes(v, Load(d16, kIdx));
return LowerHalf(LowerHalf(v16));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec64<uint32_t> TruncateTo(D , Vec128<uint64_t> v) {
return Vec64<uint32_t>{_mm_shuffle_epi32(v.raw, 0x88)};
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TruncateTo(D , VFromD<Rebind<uint32_t, D>> v) {
const DFromV<decltype(v)> du32;
const Repartition<uint8_t, decltype(du32)> d;
alignas(16) static constexpr uint8_t kIdx[16] = {
0x0u, 0x4u, 0x8u, 0xCu, 0x0u, 0x4u, 0x8u, 0xCu,
0x0u, 0x4u, 0x8u, 0xCu, 0x0u, 0x4u, 0x8u, 0xCu};
return LowerHalf(LowerHalf(TableLookupBytes(v, Load(d, kIdx))));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TruncateTo(D , VFromD<Rebind<uint32_t, D>> v) {
const DFromV<decltype(v)> du32;
const Repartition<uint16_t, decltype(du32)> d;
return LowerHalf(ConcatEven(d, BitCast(d, v), BitCast(d, v)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TruncateTo(D , VFromD<Rebind<uint16_t, D>> v) {
const DFromV<decltype(v)> du16;
const Repartition<uint8_t, decltype(du16)> d;
return LowerHalf(ConcatEven(d, BitCast(d, v), BitCast(d, v)));
}
namespace detail {
template <class D, hwy::EnableIf<!IsSigned<TFromD<D> >()>* = nullptr>
inline __attribute__((always_inline)) VFromD<Rebind<uint64_t, D>> DemoteFromU64MaskOutResult(
D , VFromD<Rebind<uint64_t, D>> v) {
return v;
}
template <class D, hwy::EnableIf<IsSigned<TFromD<D> >() && !IsFloat<TFromD<D> >() && !IsSpecialFloat<TFromD<D> >()>* = nullptr>
inline __attribute__((always_inline)) VFromD<Rebind<uint64_t, D>> DemoteFromU64MaskOutResult(
D , VFromD<Rebind<uint64_t, D>> v) {
const DFromV<decltype(v)> du64;
return And(v,
Set(du64, static_cast<uint64_t>(hwy::HighestValue<TFromD<D>>())));
}
template <class D>
inline __attribute__((always_inline)) VFromD<Rebind<uint64_t, D>> DemoteFromU64Saturate(
D dn, VFromD<Rebind<uint64_t, D>> v) {
const Rebind<uint64_t, D> du64;
const RebindToSigned<decltype(du64)> di64;
constexpr int kShiftAmt = static_cast<int>(sizeof(TFromD<D>) * 8) -
static_cast<int>(hwy::IsSigned<TFromD<D>>());
const auto too_big = BitCast(
du64, VecFromMask(
di64, Gt(BitCast(di64, ShiftRight<kShiftAmt>(v)), Zero(di64))));
return DemoteFromU64MaskOutResult(dn, Or(v, too_big));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, class V>
inline __attribute__((always_inline)) VFromD<D> ReorderDemote2From64To32Combine(D dn, V a, V b) {
return ConcatEven(dn, BitCast(dn, b), BitCast(dn, a));
}
}
template <class D, hwy::EnableIf<((size_t{1} << sizeof(TFromD<D>)) & ((1 << 1) | (1 << 2) | (1 << 4))) != 0>* = nullptr,
hwy::EnableIf<IsSigned<TFromD<D> >() && !IsFloat<TFromD<D> >() && !IsSpecialFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D dn, VFromD<Rebind<int64_t, D>> v) {
const DFromV<decltype(v)> di64;
const RebindToUnsigned<decltype(di64)> du64;
const RebindToUnsigned<decltype(dn)> dn_u;
const auto invert_mask = BitCast(du64, BroadcastSignBit(v));
const auto saturated_vals = Xor(
invert_mask,
detail::DemoteFromU64Saturate(dn, Xor(invert_mask, BitCast(du64, v))));
return BitCast(dn, TruncateTo(dn_u, saturated_vals));
}
template <class D, hwy::EnableIf<((size_t{1} << sizeof(TFromD<D>)) & ((1 << 1) | (1 << 2) | (1 << 4))) != 0>* = nullptr,
hwy::EnableIf<!IsSigned<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D dn, VFromD<Rebind<int64_t, D>> v) {
const DFromV<decltype(v)> di64;
const RebindToUnsigned<decltype(di64)> du64;
const auto non_neg_vals = BitCast(du64, AndNot(BroadcastSignBit(v), v));
return TruncateTo(dn, detail::DemoteFromU64Saturate(dn, non_neg_vals));
}
template <class D, hwy::EnableIf<((size_t{1} << sizeof(TFromD<D>)) & ((1 << 1) | (1 << 2) | (1 << 4))) != 0>* = nullptr,
hwy::EnableIf<!IsSigned<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D dn, VFromD<Rebind<uint64_t, D>> v) {
return TruncateTo(dn, detail::DemoteFromU64Saturate(dn, v));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16 / 2>* = nullptr,
hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr, hwy::EnableIf<!hwy::IsFloat<TFromD<D> >() && !hwy::IsSpecialFloat<TFromD<D> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<int64_t, D>> a,
VFromD<Repartition<int64_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16 / 2>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderDemote2To(D dn, VFromD<Repartition<uint64_t, D>> a,
VFromD<Repartition<uint64_t, D>> b) {
const DFromV<decltype(a)> d;
const Twice<decltype(d)> dt;
return DemoteTo(dn, Combine(dt, b, a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t> ReorderDemote2To(D dn, Vec128<int64_t> a,
Vec128<int64_t> b) {
const DFromV<decltype(a)> di64;
const RebindToUnsigned<decltype(di64)> du64;
const Half<decltype(dn)> dnh;
const auto invert_mask_a = BitCast(du64, BroadcastSignBit(a));
const auto invert_mask_b = BitCast(du64, BroadcastSignBit(b));
const auto saturated_a = Xor(
invert_mask_a,
detail::DemoteFromU64Saturate(dnh, Xor(invert_mask_a, BitCast(du64, a))));
const auto saturated_b = Xor(
invert_mask_b,
detail::DemoteFromU64Saturate(dnh, Xor(invert_mask_b, BitCast(du64, b))));
return ConcatEven(dn, BitCast(dn, saturated_b), BitCast(dn, saturated_a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> ReorderDemote2To(D dn, Vec128<int64_t> a,
Vec128<int64_t> b) {
const DFromV<decltype(a)> di64;
const RebindToUnsigned<decltype(di64)> du64;
const Half<decltype(dn)> dnh;
const auto saturated_a = detail::DemoteFromU64Saturate(
dnh, BitCast(du64, AndNot(BroadcastSignBit(a), a)));
const auto saturated_b = detail::DemoteFromU64Saturate(
dnh, BitCast(du64, AndNot(BroadcastSignBit(b), b)));
return ConcatEven(dn, BitCast(dn, saturated_b), BitCast(dn, saturated_a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint32_t> ReorderDemote2To(D dn, Vec128<uint64_t> a,
Vec128<uint64_t> b) {
const Half<decltype(dn)> dnh;
const auto saturated_a = detail::DemoteFromU64Saturate(dnh, a);
const auto saturated_b = detail::DemoteFromU64Saturate(dnh, b);
return ConcatEven(dn, BitCast(dn, saturated_b), BitCast(dn, saturated_a));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConvertTo(D , VFromD<Rebind<int32_t, D>> v) {
return VFromD<D>{_mm_cvtepi32_ps(v.raw)};
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConvertTo(D df, VFromD<Rebind<uint32_t, D>> v) {
const RebindToUnsigned<decltype(df)> du32;
const RebindToSigned<decltype(df)> d32;
const auto msk_lo = Set(du32, 0xFFFF);
const auto cnst2_16_flt = Set(df, 65536.0f);
const auto v_lo = BitCast(d32, And(v, msk_lo));
const auto v_hi = BitCast(d32, ShiftRight<16>(v));
return MulAdd(cnst2_16_flt, ConvertTo(df, v_hi), ConvertTo(df, v_lo));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConvertTo(D dd, VFromD<Rebind<int64_t, D>> v) {
const Repartition<uint32_t, decltype(dd)> d32;
const Repartition<uint64_t, decltype(dd)> d64;
const auto k84_63 = Set(d64, 0x4530000080000000ULL);
const auto v_upper = BitCast(dd, ShiftRight<32>(BitCast(d64, v)) ^ k84_63);
const auto k52 = Set(d32, 0x43300000);
const auto v_lower = BitCast(dd, OddEven(k52, BitCast(d32, v)));
const auto k84_63_52 = BitCast(dd, Set(d64, 0x4530000080100000ULL));
return (v_upper - k84_63_52) + v_lower;
}
namespace detail {
template <class VW>
inline __attribute__((always_inline)) VFromD<Rebind<double, DFromV<VW>>> U64ToF64VecFast(VW w) {
const DFromV<decltype(w)> d64;
const RebindToFloat<decltype(d64)> dd;
const auto cnst2_52_dbl = Set(dd, 0x0010000000000000);
return BitCast(dd, Or(w, BitCast(d64, cnst2_52_dbl))) - cnst2_52_dbl;
}
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConvertTo(D dd, VFromD<Rebind<uint64_t, D>> v) {
const RebindToUnsigned<decltype(dd)> d64;
using VU = VFromD<decltype(d64)>;
const VU msk_lo = Set(d64, 0xFFFFFFFF);
const auto cnst2_32_dbl = Set(dd, 4294967296.0);
const VU v_lo = And(v, msk_lo);
const VU v_hi = ShiftRight<32>(v);
const auto v_lo_dbl = detail::U64ToF64VecFast(v_lo);
return MulAdd(cnst2_32_dbl, detail::U64ToF64VecFast(v_hi), v_lo_dbl);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConvertTo(D di, VFromD<Rebind<float, D>> v) {
return detail::FixConversionOverflow(di, v, _mm_cvttps_epi32(v.raw));
}
template <class DI, hwy::EnableIf<(DI::kPrivateLanes * sizeof(TFromD<DI>) > (0 ? 16 : 0))>* = nullptr,
hwy::EnableIf<IsSame<TFromD<DI>, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DI> ConvertTo(DI di, VFromD<Rebind<double, DI>> v) {
using VI = VFromD<decltype(di)>;
const RebindToUnsigned<decltype(di)> du;
using VU = VFromD<decltype(du)>;
const Repartition<uint16_t, decltype(di)> du16;
const VI k1075 = Set(di, 1075);
const VU biased_exp = ShiftRight<52>(BitCast(du, v)) & Set(du, 0x7FF);
const auto in_range = BitCast(di, biased_exp) < Set(di, 1086);
const VU shift_mnt = BitCast(
du, SaturatedSub(BitCast(du16, k1075), BitCast(du16, biased_exp)));
const VU shift_int = BitCast(
du, SaturatedSub(BitCast(du16, biased_exp), BitCast(du16, k1075)));
const VU mantissa = BitCast(du, v) & Set(du, (1ULL << 52) - 1);
const VU int53 = (mantissa | Set(du, 1ULL << 52)) >> shift_mnt;
const VU shifted = int53 << shift_int;
const VI sign_mask = BroadcastSignBit(BitCast(di, v));
const VI limit = Set(di, LimitsMax<int64_t>()) - sign_mask;
const VI magnitude = IfThenElse(in_range, BitCast(di, shifted), limit);
return (magnitude ^ sign_mask) - sign_mask;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> NearestInt(const Vec128<float, N> v) {
const RebindToSigned<DFromV<decltype(v)>> di;
return detail::FixConversionOverflow(di, v, _mm_cvtps_epi32(v.raw));
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Round(const Vec128<float, N> v) {
return Vec128<float, N>{
_mm_round_ps(v.raw,
0x00
|
0x08
)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Round(const Vec128<double, N> v) {
return Vec128<double, N>{
_mm_round_pd(v.raw,
0x00
|
0x08
)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Trunc(const Vec128<float, N> v) {
return Vec128<float, N>{
_mm_round_ps(v.raw,
0x03
|
0x08
)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Trunc(const Vec128<double, N> v) {
return Vec128<double, N>{
_mm_round_pd(v.raw,
0x03
|
0x08
)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Ceil(const Vec128<float, N> v) {
return Vec128<float, N>{
_mm_round_ps(v.raw,
0x02
|
0x08
)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Ceil(const Vec128<double, N> v) {
return Vec128<double, N>{
_mm_round_pd(v.raw,
0x02
|
0x08
)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> Floor(const Vec128<float, N> v) {
return Vec128<float, N>{
_mm_round_ps(v.raw,
0x01
|
0x08
)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<double, N> Floor(const Vec128<double, N> v) {
return Vec128<double, N>{
_mm_round_pd(v.raw,
0x01
|
0x08
)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<float, N> IsNaN(const Vec128<float, N> v) {
return Mask128<float, N>{_mm_cmpunord_ps(v.raw, v.raw)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<double, N> IsNaN(const Vec128<double, N> v) {
return Mask128<double, N>{_mm_cmpunord_pd(v.raw, v.raw)};
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> IsInf(const Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> d;
const RebindToSigned<decltype(d)> di;
const VFromD<decltype(di)> vi = BitCast(di, v);
return RebindMask(d, Eq(Add(vi, vi), Set(di, hwy::MaxExponentTimes2<T>())));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> IsFinite(const Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const RebindToSigned<decltype(d)> di;
const VFromD<decltype(du)> vu = BitCast(du, v);
const VFromD<decltype(di)> exp =
BitCast(di, ShiftRight<hwy::MantissaBits<T>() + 1>(ShiftLeft<1>(vu)));
return RebindMask(d, Lt(exp, Set(di, hwy::MaxExponentField<T>())));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t> AESRound(Vec128<uint8_t> state,
Vec128<uint8_t> round_key) {
return Vec128<uint8_t>{_mm_aesenc_si128(state.raw, round_key.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t> AESLastRound(Vec128<uint8_t> state,
Vec128<uint8_t> round_key) {
return Vec128<uint8_t>{_mm_aesenclast_si128(state.raw, round_key.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t> AESInvMixColumns(Vec128<uint8_t> state) {
return Vec128<uint8_t>{_mm_aesimc_si128(state.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t> AESRoundInv(Vec128<uint8_t> state,
Vec128<uint8_t> round_key) {
return Vec128<uint8_t>{_mm_aesdec_si128(state.raw, round_key.raw)};
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t> AESLastRoundInv(Vec128<uint8_t> state,
Vec128<uint8_t> round_key) {
return Vec128<uint8_t>{_mm_aesdeclast_si128(state.raw, round_key.raw)};
}
template <uint8_t kRcon>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t> AESKeyGenAssist(Vec128<uint8_t> v) {
return Vec128<uint8_t>{_mm_aeskeygenassist_si128(v.raw, kRcon)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> CLMulLower(Vec128<uint64_t, N> a,
Vec128<uint64_t, N> b) {
return Vec128<uint64_t, N>{_mm_clmulepi64_si128(a.raw, b.raw, 0x00)};
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, N> CLMulUpper(Vec128<uint64_t, N> a,
Vec128<uint64_t, N> b) {
return Vec128<uint64_t, N>{_mm_clmulepi64_si128(a.raw, b.raw, 0x11)};
}
namespace detail {
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
inline __attribute__((always_inline)) MFromD<D> LoadMaskBits128(D d, uint64_t mask_bits) {
const RebindToUnsigned<decltype(d)> du;
const VFromD<D> vbits{_mm_cvtsi32_si128(static_cast<int>(mask_bits))};
alignas(16) static constexpr uint8_t kRep8[16] = {0, 0, 0, 0, 0, 0, 0, 0,
1, 1, 1, 1, 1, 1, 1, 1};
const auto rep8 = TableLookupBytes(vbits, Load(du, kRep8));
alignas(16) static constexpr uint8_t kBit[16] = {1, 2, 4, 8, 16, 32, 64, 128,
1, 2, 4, 8, 16, 32, 64, 128};
return RebindMask(d, TestBit(rep8, LoadDup128(du, kBit)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
inline __attribute__((always_inline)) MFromD<D> LoadMaskBits128(D d, uint64_t mask_bits) {
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint16_t kBit[8] = {1, 2, 4, 8, 16, 32, 64, 128};
const auto vmask_bits = Set(du, static_cast<uint16_t>(mask_bits));
return RebindMask(d, TestBit(vmask_bits, Load(du, kBit)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
inline __attribute__((always_inline)) MFromD<D> LoadMaskBits128(D d, uint64_t mask_bits) {
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint32_t kBit[8] = {1, 2, 4, 8};
const auto vmask_bits = Set(du, static_cast<uint32_t>(mask_bits));
return RebindMask(d, TestBit(vmask_bits, Load(du, kBit)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
inline __attribute__((always_inline)) MFromD<D> LoadMaskBits128(D d, uint64_t mask_bits) {
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint64_t kBit[8] = {1, 2};
return RebindMask(d, TestBit(Set(du, mask_bits), Load(du, kBit)));
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> LoadMaskBits(D d, const uint8_t* __restrict__ bits) {
constexpr size_t kN = MaxLanes(d);
uint64_t mask_bits = 0;
constexpr size_t kNumBytes = (kN + 7) / 8;
CopyBytes<kNumBytes>(bits, &mask_bits);
if (kN < 8) {
mask_bits &= (1ull << kN) - 1;
}
return detail::LoadMaskBits128(d, mask_bits);
}
template <typename T>
struct CompressIsPartition {
enum { value = (sizeof(T) != 1) };
};
namespace detail {
constexpr inline __attribute__((always_inline)) uint64_t U64FromInt(int mask_bits) {
return static_cast<uint64_t>(static_cast<unsigned>(mask_bits));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) uint64_t BitsFromMask(hwy::SizeTag<1> ,
const Mask128<T, N> mask) {
const Simd<T, N, 0> d;
const auto sign_bits = BitCast(d, VecFromMask(d, mask)).raw;
return U64FromInt(_mm_movemask_epi8(sign_bits));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) uint64_t BitsFromMask(hwy::SizeTag<2> ,
const Mask128<T, N> mask) {
const auto sign_bits = _mm_packs_epi16(mask.raw, _mm_setzero_si128());
return U64FromInt(_mm_movemask_epi8(sign_bits));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) uint64_t BitsFromMask(hwy::SizeTag<4> , Mask128<T, N> mask) {
const Simd<T, N, 0> d;
const Simd<float, N, 0> df;
const auto sign_bits = BitCast(df, VecFromMask(d, mask));
return U64FromInt(_mm_movemask_ps(sign_bits.raw));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) uint64_t BitsFromMask(hwy::SizeTag<8> , Mask128<T, N> mask) {
const Simd<T, N, 0> d;
const Simd<double, N, 0> df;
const auto sign_bits = BitCast(df, VecFromMask(d, mask));
return U64FromInt(_mm_movemask_pd(sign_bits.raw));
}
template <typename T, size_t N>
inline __attribute__((always_inline)) uint64_t BitsFromMask(const Mask128<T, N> mask) {
return OnlyActive<T, N>(BitsFromMask(hwy::SizeTag<sizeof(T)>(), mask));
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t StoreMaskBits(D d, MFromD<D> mask, uint8_t* bits) {
constexpr size_t kNumBytes = (MaxLanes(d) + 7) / 8;
const uint64_t mask_bits = detail::BitsFromMask(mask);
CopyBytes<kNumBytes>(&mask_bits, bits);
return kNumBytes;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) bool AllFalse(D , MFromD<D> mask) {
return detail::BitsFromMask(mask) == 0;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) bool AllTrue(D d, MFromD<D> mask) {
constexpr uint64_t kAllBits = (1ull << MaxLanes(d)) - 1;
return detail::BitsFromMask(mask) == kAllBits;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CountTrue(D , MFromD<D> mask) {
return PopCount(detail::BitsFromMask(mask));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t FindKnownFirstTrue(D , MFromD<D> mask) {
return Num0BitsBelowLS1Bit_Nonzero32(
static_cast<uint32_t>(detail::BitsFromMask(mask)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) intptr_t FindFirstTrue(D , MFromD<D> mask) {
const uint32_t mask_bits = static_cast<uint32_t>(detail::BitsFromMask(mask));
return mask_bits ? intptr_t(Num0BitsBelowLS1Bit_Nonzero32(mask_bits)) : -1;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t FindKnownLastTrue(D , MFromD<D> mask) {
return 31 - Num0BitsAboveMS1Bit_Nonzero32(
static_cast<uint32_t>(detail::BitsFromMask(mask)));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) intptr_t FindLastTrue(D , MFromD<D> mask) {
const uint32_t mask_bits = static_cast<uint32_t>(detail::BitsFromMask(mask));
return mask_bits ? intptr_t(31 - Num0BitsAboveMS1Bit_Nonzero32(mask_bits))
: -1;
}
namespace detail {
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromBits128(D d, uint64_t mask_bits) {
do { } while (0);
const Rebind<uint8_t, decltype(d)> d8;
const Twice<decltype(d8)> d8t;
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint8_t table[2048] = {
0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
2, 0, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
4, 0, 2, 6, 8, 10, 12, 14, 0, 4, 2, 6, 8, 10, 12, 14,
2, 4, 0, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
6, 0, 2, 4, 8, 10, 12, 14, 0, 6, 2, 4, 8, 10, 12, 14,
2, 6, 0, 4, 8, 10, 12, 14, 0, 2, 6, 4, 8, 10, 12, 14,
4, 6, 0, 2, 8, 10, 12, 14, 0, 4, 6, 2, 8, 10, 12, 14,
2, 4, 6, 0, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
8, 0, 2, 4, 6, 10, 12, 14, 0, 8, 2, 4, 6, 10, 12, 14,
2, 8, 0, 4, 6, 10, 12, 14, 0, 2, 8, 4, 6, 10, 12, 14,
4, 8, 0, 2, 6, 10, 12, 14, 0, 4, 8, 2, 6, 10, 12, 14,
2, 4, 8, 0, 6, 10, 12, 14, 0, 2, 4, 8, 6, 10, 12, 14,
6, 8, 0, 2, 4, 10, 12, 14, 0, 6, 8, 2, 4, 10, 12, 14,
2, 6, 8, 0, 4, 10, 12, 14, 0, 2, 6, 8, 4, 10, 12, 14,
4, 6, 8, 0, 2, 10, 12, 14, 0, 4, 6, 8, 2, 10, 12, 14,
2, 4, 6, 8, 0, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
10, 0, 2, 4, 6, 8, 12, 14, 0, 10, 2, 4, 6, 8, 12, 14,
2, 10, 0, 4, 6, 8, 12, 14, 0, 2, 10, 4, 6, 8, 12, 14,
4, 10, 0, 2, 6, 8, 12, 14, 0, 4, 10, 2, 6, 8, 12, 14,
2, 4, 10, 0, 6, 8, 12, 14, 0, 2, 4, 10, 6, 8, 12, 14,
6, 10, 0, 2, 4, 8, 12, 14, 0, 6, 10, 2, 4, 8, 12, 14,
2, 6, 10, 0, 4, 8, 12, 14, 0, 2, 6, 10, 4, 8, 12, 14,
4, 6, 10, 0, 2, 8, 12, 14, 0, 4, 6, 10, 2, 8, 12, 14,
2, 4, 6, 10, 0, 8, 12, 14, 0, 2, 4, 6, 10, 8, 12, 14,
8, 10, 0, 2, 4, 6, 12, 14, 0, 8, 10, 2, 4, 6, 12, 14,
2, 8, 10, 0, 4, 6, 12, 14, 0, 2, 8, 10, 4, 6, 12, 14,
4, 8, 10, 0, 2, 6, 12, 14, 0, 4, 8, 10, 2, 6, 12, 14,
2, 4, 8, 10, 0, 6, 12, 14, 0, 2, 4, 8, 10, 6, 12, 14,
6, 8, 10, 0, 2, 4, 12, 14, 0, 6, 8, 10, 2, 4, 12, 14,
2, 6, 8, 10, 0, 4, 12, 14, 0, 2, 6, 8, 10, 4, 12, 14,
4, 6, 8, 10, 0, 2, 12, 14, 0, 4, 6, 8, 10, 2, 12, 14,
2, 4, 6, 8, 10, 0, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14,
12, 0, 2, 4, 6, 8, 10, 14, 0, 12, 2, 4, 6, 8, 10, 14,
2, 12, 0, 4, 6, 8, 10, 14, 0, 2, 12, 4, 6, 8, 10, 14,
4, 12, 0, 2, 6, 8, 10, 14, 0, 4, 12, 2, 6, 8, 10, 14,
2, 4, 12, 0, 6, 8, 10, 14, 0, 2, 4, 12, 6, 8, 10, 14,
6, 12, 0, 2, 4, 8, 10, 14, 0, 6, 12, 2, 4, 8, 10, 14,
2, 6, 12, 0, 4, 8, 10, 14, 0, 2, 6, 12, 4, 8, 10, 14,
4, 6, 12, 0, 2, 8, 10, 14, 0, 4, 6, 12, 2, 8, 10, 14,
2, 4, 6, 12, 0, 8, 10, 14, 0, 2, 4, 6, 12, 8, 10, 14,
8, 12, 0, 2, 4, 6, 10, 14, 0, 8, 12, 2, 4, 6, 10, 14,
2, 8, 12, 0, 4, 6, 10, 14, 0, 2, 8, 12, 4, 6, 10, 14,
4, 8, 12, 0, 2, 6, 10, 14, 0, 4, 8, 12, 2, 6, 10, 14,
2, 4, 8, 12, 0, 6, 10, 14, 0, 2, 4, 8, 12, 6, 10, 14,
6, 8, 12, 0, 2, 4, 10, 14, 0, 6, 8, 12, 2, 4, 10, 14,
2, 6, 8, 12, 0, 4, 10, 14, 0, 2, 6, 8, 12, 4, 10, 14,
4, 6, 8, 12, 0, 2, 10, 14, 0, 4, 6, 8, 12, 2, 10, 14,
2, 4, 6, 8, 12, 0, 10, 14, 0, 2, 4, 6, 8, 12, 10, 14,
10, 12, 0, 2, 4, 6, 8, 14, 0, 10, 12, 2, 4, 6, 8, 14,
2, 10, 12, 0, 4, 6, 8, 14, 0, 2, 10, 12, 4, 6, 8, 14,
4, 10, 12, 0, 2, 6, 8, 14, 0, 4, 10, 12, 2, 6, 8, 14,
2, 4, 10, 12, 0, 6, 8, 14, 0, 2, 4, 10, 12, 6, 8, 14,
6, 10, 12, 0, 2, 4, 8, 14, 0, 6, 10, 12, 2, 4, 8, 14,
2, 6, 10, 12, 0, 4, 8, 14, 0, 2, 6, 10, 12, 4, 8, 14,
4, 6, 10, 12, 0, 2, 8, 14, 0, 4, 6, 10, 12, 2, 8, 14,
2, 4, 6, 10, 12, 0, 8, 14, 0, 2, 4, 6, 10, 12, 8, 14,
8, 10, 12, 0, 2, 4, 6, 14, 0, 8, 10, 12, 2, 4, 6, 14,
2, 8, 10, 12, 0, 4, 6, 14, 0, 2, 8, 10, 12, 4, 6, 14,
4, 8, 10, 12, 0, 2, 6, 14, 0, 4, 8, 10, 12, 2, 6, 14,
2, 4, 8, 10, 12, 0, 6, 14, 0, 2, 4, 8, 10, 12, 6, 14,
6, 8, 10, 12, 0, 2, 4, 14, 0, 6, 8, 10, 12, 2, 4, 14,
2, 6, 8, 10, 12, 0, 4, 14, 0, 2, 6, 8, 10, 12, 4, 14,
4, 6, 8, 10, 12, 0, 2, 14, 0, 4, 6, 8, 10, 12, 2, 14,
2, 4, 6, 8, 10, 12, 0, 14, 0, 2, 4, 6, 8, 10, 12, 14,
14, 0, 2, 4, 6, 8, 10, 12, 0, 14, 2, 4, 6, 8, 10, 12,
2, 14, 0, 4, 6, 8, 10, 12, 0, 2, 14, 4, 6, 8, 10, 12,
4, 14, 0, 2, 6, 8, 10, 12, 0, 4, 14, 2, 6, 8, 10, 12,
2, 4, 14, 0, 6, 8, 10, 12, 0, 2, 4, 14, 6, 8, 10, 12,
6, 14, 0, 2, 4, 8, 10, 12, 0, 6, 14, 2, 4, 8, 10, 12,
2, 6, 14, 0, 4, 8, 10, 12, 0, 2, 6, 14, 4, 8, 10, 12,
4, 6, 14, 0, 2, 8, 10, 12, 0, 4, 6, 14, 2, 8, 10, 12,
2, 4, 6, 14, 0, 8, 10, 12, 0, 2, 4, 6, 14, 8, 10, 12,
8, 14, 0, 2, 4, 6, 10, 12, 0, 8, 14, 2, 4, 6, 10, 12,
2, 8, 14, 0, 4, 6, 10, 12, 0, 2, 8, 14, 4, 6, 10, 12,
4, 8, 14, 0, 2, 6, 10, 12, 0, 4, 8, 14, 2, 6, 10, 12,
2, 4, 8, 14, 0, 6, 10, 12, 0, 2, 4, 8, 14, 6, 10, 12,
6, 8, 14, 0, 2, 4, 10, 12, 0, 6, 8, 14, 2, 4, 10, 12,
2, 6, 8, 14, 0, 4, 10, 12, 0, 2, 6, 8, 14, 4, 10, 12,
4, 6, 8, 14, 0, 2, 10, 12, 0, 4, 6, 8, 14, 2, 10, 12,
2, 4, 6, 8, 14, 0, 10, 12, 0, 2, 4, 6, 8, 14, 10, 12,
10, 14, 0, 2, 4, 6, 8, 12, 0, 10, 14, 2, 4, 6, 8, 12,
2, 10, 14, 0, 4, 6, 8, 12, 0, 2, 10, 14, 4, 6, 8, 12,
4, 10, 14, 0, 2, 6, 8, 12, 0, 4, 10, 14, 2, 6, 8, 12,
2, 4, 10, 14, 0, 6, 8, 12, 0, 2, 4, 10, 14, 6, 8, 12,
6, 10, 14, 0, 2, 4, 8, 12, 0, 6, 10, 14, 2, 4, 8, 12,
2, 6, 10, 14, 0, 4, 8, 12, 0, 2, 6, 10, 14, 4, 8, 12,
4, 6, 10, 14, 0, 2, 8, 12, 0, 4, 6, 10, 14, 2, 8, 12,
2, 4, 6, 10, 14, 0, 8, 12, 0, 2, 4, 6, 10, 14, 8, 12,
8, 10, 14, 0, 2, 4, 6, 12, 0, 8, 10, 14, 2, 4, 6, 12,
2, 8, 10, 14, 0, 4, 6, 12, 0, 2, 8, 10, 14, 4, 6, 12,
4, 8, 10, 14, 0, 2, 6, 12, 0, 4, 8, 10, 14, 2, 6, 12,
2, 4, 8, 10, 14, 0, 6, 12, 0, 2, 4, 8, 10, 14, 6, 12,
6, 8, 10, 14, 0, 2, 4, 12, 0, 6, 8, 10, 14, 2, 4, 12,
2, 6, 8, 10, 14, 0, 4, 12, 0, 2, 6, 8, 10, 14, 4, 12,
4, 6, 8, 10, 14, 0, 2, 12, 0, 4, 6, 8, 10, 14, 2, 12,
2, 4, 6, 8, 10, 14, 0, 12, 0, 2, 4, 6, 8, 10, 14, 12,
12, 14, 0, 2, 4, 6, 8, 10, 0, 12, 14, 2, 4, 6, 8, 10,
2, 12, 14, 0, 4, 6, 8, 10, 0, 2, 12, 14, 4, 6, 8, 10,
4, 12, 14, 0, 2, 6, 8, 10, 0, 4, 12, 14, 2, 6, 8, 10,
2, 4, 12, 14, 0, 6, 8, 10, 0, 2, 4, 12, 14, 6, 8, 10,
6, 12, 14, 0, 2, 4, 8, 10, 0, 6, 12, 14, 2, 4, 8, 10,
2, 6, 12, 14, 0, 4, 8, 10, 0, 2, 6, 12, 14, 4, 8, 10,
4, 6, 12, 14, 0, 2, 8, 10, 0, 4, 6, 12, 14, 2, 8, 10,
2, 4, 6, 12, 14, 0, 8, 10, 0, 2, 4, 6, 12, 14, 8, 10,
8, 12, 14, 0, 2, 4, 6, 10, 0, 8, 12, 14, 2, 4, 6, 10,
2, 8, 12, 14, 0, 4, 6, 10, 0, 2, 8, 12, 14, 4, 6, 10,
4, 8, 12, 14, 0, 2, 6, 10, 0, 4, 8, 12, 14, 2, 6, 10,
2, 4, 8, 12, 14, 0, 6, 10, 0, 2, 4, 8, 12, 14, 6, 10,
6, 8, 12, 14, 0, 2, 4, 10, 0, 6, 8, 12, 14, 2, 4, 10,
2, 6, 8, 12, 14, 0, 4, 10, 0, 2, 6, 8, 12, 14, 4, 10,
4, 6, 8, 12, 14, 0, 2, 10, 0, 4, 6, 8, 12, 14, 2, 10,
2, 4, 6, 8, 12, 14, 0, 10, 0, 2, 4, 6, 8, 12, 14, 10,
10, 12, 14, 0, 2, 4, 6, 8, 0, 10, 12, 14, 2, 4, 6, 8,
2, 10, 12, 14, 0, 4, 6, 8, 0, 2, 10, 12, 14, 4, 6, 8,
4, 10, 12, 14, 0, 2, 6, 8, 0, 4, 10, 12, 14, 2, 6, 8,
2, 4, 10, 12, 14, 0, 6, 8, 0, 2, 4, 10, 12, 14, 6, 8,
6, 10, 12, 14, 0, 2, 4, 8, 0, 6, 10, 12, 14, 2, 4, 8,
2, 6, 10, 12, 14, 0, 4, 8, 0, 2, 6, 10, 12, 14, 4, 8,
4, 6, 10, 12, 14, 0, 2, 8, 0, 4, 6, 10, 12, 14, 2, 8,
2, 4, 6, 10, 12, 14, 0, 8, 0, 2, 4, 6, 10, 12, 14, 8,
8, 10, 12, 14, 0, 2, 4, 6, 0, 8, 10, 12, 14, 2, 4, 6,
2, 8, 10, 12, 14, 0, 4, 6, 0, 2, 8, 10, 12, 14, 4, 6,
4, 8, 10, 12, 14, 0, 2, 6, 0, 4, 8, 10, 12, 14, 2, 6,
2, 4, 8, 10, 12, 14, 0, 6, 0, 2, 4, 8, 10, 12, 14, 6,
6, 8, 10, 12, 14, 0, 2, 4, 0, 6, 8, 10, 12, 14, 2, 4,
2, 6, 8, 10, 12, 14, 0, 4, 0, 2, 6, 8, 10, 12, 14, 4,
4, 6, 8, 10, 12, 14, 0, 2, 0, 4, 6, 8, 10, 12, 14, 2,
2, 4, 6, 8, 10, 12, 14, 0, 0, 2, 4, 6, 8, 10, 12, 14};
const VFromD<decltype(d8t)> byte_idx{Load(d8, table + mask_bits * 8).raw};
const VFromD<decltype(du)> pairs = ZipLower(byte_idx, byte_idx);
return BitCast(d, pairs + Set(du, 0x0100));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromNotBits128(D d, uint64_t mask_bits) {
do { } while (0);
const Rebind<uint8_t, decltype(d)> d8;
const Twice<decltype(d8)> d8t;
const RebindToUnsigned<decltype(d)> du;
alignas(16) static constexpr uint8_t table[2048] = {
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0,
0, 4, 6, 8, 10, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2,
0, 2, 6, 8, 10, 12, 14, 4, 2, 6, 8, 10, 12, 14, 0, 4,
0, 6, 8, 10, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4,
0, 2, 4, 8, 10, 12, 14, 6, 2, 4, 8, 10, 12, 14, 0, 6,
0, 4, 8, 10, 12, 14, 2, 6, 4, 8, 10, 12, 14, 0, 2, 6,
0, 2, 8, 10, 12, 14, 4, 6, 2, 8, 10, 12, 14, 0, 4, 6,
0, 8, 10, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6,
0, 2, 4, 6, 10, 12, 14, 8, 2, 4, 6, 10, 12, 14, 0, 8,
0, 4, 6, 10, 12, 14, 2, 8, 4, 6, 10, 12, 14, 0, 2, 8,
0, 2, 6, 10, 12, 14, 4, 8, 2, 6, 10, 12, 14, 0, 4, 8,
0, 6, 10, 12, 14, 2, 4, 8, 6, 10, 12, 14, 0, 2, 4, 8,
0, 2, 4, 10, 12, 14, 6, 8, 2, 4, 10, 12, 14, 0, 6, 8,
0, 4, 10, 12, 14, 2, 6, 8, 4, 10, 12, 14, 0, 2, 6, 8,
0, 2, 10, 12, 14, 4, 6, 8, 2, 10, 12, 14, 0, 4, 6, 8,
0, 10, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8,
0, 2, 4, 6, 8, 12, 14, 10, 2, 4, 6, 8, 12, 14, 0, 10,
0, 4, 6, 8, 12, 14, 2, 10, 4, 6, 8, 12, 14, 0, 2, 10,
0, 2, 6, 8, 12, 14, 4, 10, 2, 6, 8, 12, 14, 0, 4, 10,
0, 6, 8, 12, 14, 2, 4, 10, 6, 8, 12, 14, 0, 2, 4, 10,
0, 2, 4, 8, 12, 14, 6, 10, 2, 4, 8, 12, 14, 0, 6, 10,
0, 4, 8, 12, 14, 2, 6, 10, 4, 8, 12, 14, 0, 2, 6, 10,
0, 2, 8, 12, 14, 4, 6, 10, 2, 8, 12, 14, 0, 4, 6, 10,
0, 8, 12, 14, 2, 4, 6, 10, 8, 12, 14, 0, 2, 4, 6, 10,
0, 2, 4, 6, 12, 14, 8, 10, 2, 4, 6, 12, 14, 0, 8, 10,
0, 4, 6, 12, 14, 2, 8, 10, 4, 6, 12, 14, 0, 2, 8, 10,
0, 2, 6, 12, 14, 4, 8, 10, 2, 6, 12, 14, 0, 4, 8, 10,
0, 6, 12, 14, 2, 4, 8, 10, 6, 12, 14, 0, 2, 4, 8, 10,
0, 2, 4, 12, 14, 6, 8, 10, 2, 4, 12, 14, 0, 6, 8, 10,
0, 4, 12, 14, 2, 6, 8, 10, 4, 12, 14, 0, 2, 6, 8, 10,
0, 2, 12, 14, 4, 6, 8, 10, 2, 12, 14, 0, 4, 6, 8, 10,
0, 12, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10,
0, 2, 4, 6, 8, 10, 14, 12, 2, 4, 6, 8, 10, 14, 0, 12,
0, 4, 6, 8, 10, 14, 2, 12, 4, 6, 8, 10, 14, 0, 2, 12,
0, 2, 6, 8, 10, 14, 4, 12, 2, 6, 8, 10, 14, 0, 4, 12,
0, 6, 8, 10, 14, 2, 4, 12, 6, 8, 10, 14, 0, 2, 4, 12,
0, 2, 4, 8, 10, 14, 6, 12, 2, 4, 8, 10, 14, 0, 6, 12,
0, 4, 8, 10, 14, 2, 6, 12, 4, 8, 10, 14, 0, 2, 6, 12,
0, 2, 8, 10, 14, 4, 6, 12, 2, 8, 10, 14, 0, 4, 6, 12,
0, 8, 10, 14, 2, 4, 6, 12, 8, 10, 14, 0, 2, 4, 6, 12,
0, 2, 4, 6, 10, 14, 8, 12, 2, 4, 6, 10, 14, 0, 8, 12,
0, 4, 6, 10, 14, 2, 8, 12, 4, 6, 10, 14, 0, 2, 8, 12,
0, 2, 6, 10, 14, 4, 8, 12, 2, 6, 10, 14, 0, 4, 8, 12,
0, 6, 10, 14, 2, 4, 8, 12, 6, 10, 14, 0, 2, 4, 8, 12,
0, 2, 4, 10, 14, 6, 8, 12, 2, 4, 10, 14, 0, 6, 8, 12,
0, 4, 10, 14, 2, 6, 8, 12, 4, 10, 14, 0, 2, 6, 8, 12,
0, 2, 10, 14, 4, 6, 8, 12, 2, 10, 14, 0, 4, 6, 8, 12,
0, 10, 14, 2, 4, 6, 8, 12, 10, 14, 0, 2, 4, 6, 8, 12,
0, 2, 4, 6, 8, 14, 10, 12, 2, 4, 6, 8, 14, 0, 10, 12,
0, 4, 6, 8, 14, 2, 10, 12, 4, 6, 8, 14, 0, 2, 10, 12,
0, 2, 6, 8, 14, 4, 10, 12, 2, 6, 8, 14, 0, 4, 10, 12,
0, 6, 8, 14, 2, 4, 10, 12, 6, 8, 14, 0, 2, 4, 10, 12,
0, 2, 4, 8, 14, 6, 10, 12, 2, 4, 8, 14, 0, 6, 10, 12,
0, 4, 8, 14, 2, 6, 10, 12, 4, 8, 14, 0, 2, 6, 10, 12,
0, 2, 8, 14, 4, 6, 10, 12, 2, 8, 14, 0, 4, 6, 10, 12,
0, 8, 14, 2, 4, 6, 10, 12, 8, 14, 0, 2, 4, 6, 10, 12,
0, 2, 4, 6, 14, 8, 10, 12, 2, 4, 6, 14, 0, 8, 10, 12,
0, 4, 6, 14, 2, 8, 10, 12, 4, 6, 14, 0, 2, 8, 10, 12,
0, 2, 6, 14, 4, 8, 10, 12, 2, 6, 14, 0, 4, 8, 10, 12,
0, 6, 14, 2, 4, 8, 10, 12, 6, 14, 0, 2, 4, 8, 10, 12,
0, 2, 4, 14, 6, 8, 10, 12, 2, 4, 14, 0, 6, 8, 10, 12,
0, 4, 14, 2, 6, 8, 10, 12, 4, 14, 0, 2, 6, 8, 10, 12,
0, 2, 14, 4, 6, 8, 10, 12, 2, 14, 0, 4, 6, 8, 10, 12,
0, 14, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12,
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 6, 8, 10, 12, 0, 14,
0, 4, 6, 8, 10, 12, 2, 14, 4, 6, 8, 10, 12, 0, 2, 14,
0, 2, 6, 8, 10, 12, 4, 14, 2, 6, 8, 10, 12, 0, 4, 14,
0, 6, 8, 10, 12, 2, 4, 14, 6, 8, 10, 12, 0, 2, 4, 14,
0, 2, 4, 8, 10, 12, 6, 14, 2, 4, 8, 10, 12, 0, 6, 14,
0, 4, 8, 10, 12, 2, 6, 14, 4, 8, 10, 12, 0, 2, 6, 14,
0, 2, 8, 10, 12, 4, 6, 14, 2, 8, 10, 12, 0, 4, 6, 14,
0, 8, 10, 12, 2, 4, 6, 14, 8, 10, 12, 0, 2, 4, 6, 14,
0, 2, 4, 6, 10, 12, 8, 14, 2, 4, 6, 10, 12, 0, 8, 14,
0, 4, 6, 10, 12, 2, 8, 14, 4, 6, 10, 12, 0, 2, 8, 14,
0, 2, 6, 10, 12, 4, 8, 14, 2, 6, 10, 12, 0, 4, 8, 14,
0, 6, 10, 12, 2, 4, 8, 14, 6, 10, 12, 0, 2, 4, 8, 14,
0, 2, 4, 10, 12, 6, 8, 14, 2, 4, 10, 12, 0, 6, 8, 14,
0, 4, 10, 12, 2, 6, 8, 14, 4, 10, 12, 0, 2, 6, 8, 14,
0, 2, 10, 12, 4, 6, 8, 14, 2, 10, 12, 0, 4, 6, 8, 14,
0, 10, 12, 2, 4, 6, 8, 14, 10, 12, 0, 2, 4, 6, 8, 14,
0, 2, 4, 6, 8, 12, 10, 14, 2, 4, 6, 8, 12, 0, 10, 14,
0, 4, 6, 8, 12, 2, 10, 14, 4, 6, 8, 12, 0, 2, 10, 14,
0, 2, 6, 8, 12, 4, 10, 14, 2, 6, 8, 12, 0, 4, 10, 14,
0, 6, 8, 12, 2, 4, 10, 14, 6, 8, 12, 0, 2, 4, 10, 14,
0, 2, 4, 8, 12, 6, 10, 14, 2, 4, 8, 12, 0, 6, 10, 14,
0, 4, 8, 12, 2, 6, 10, 14, 4, 8, 12, 0, 2, 6, 10, 14,
0, 2, 8, 12, 4, 6, 10, 14, 2, 8, 12, 0, 4, 6, 10, 14,
0, 8, 12, 2, 4, 6, 10, 14, 8, 12, 0, 2, 4, 6, 10, 14,
0, 2, 4, 6, 12, 8, 10, 14, 2, 4, 6, 12, 0, 8, 10, 14,
0, 4, 6, 12, 2, 8, 10, 14, 4, 6, 12, 0, 2, 8, 10, 14,
0, 2, 6, 12, 4, 8, 10, 14, 2, 6, 12, 0, 4, 8, 10, 14,
0, 6, 12, 2, 4, 8, 10, 14, 6, 12, 0, 2, 4, 8, 10, 14,
0, 2, 4, 12, 6, 8, 10, 14, 2, 4, 12, 0, 6, 8, 10, 14,
0, 4, 12, 2, 6, 8, 10, 14, 4, 12, 0, 2, 6, 8, 10, 14,
0, 2, 12, 4, 6, 8, 10, 14, 2, 12, 0, 4, 6, 8, 10, 14,
0, 12, 2, 4, 6, 8, 10, 14, 12, 0, 2, 4, 6, 8, 10, 14,
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 6, 8, 10, 0, 12, 14,
0, 4, 6, 8, 10, 2, 12, 14, 4, 6, 8, 10, 0, 2, 12, 14,
0, 2, 6, 8, 10, 4, 12, 14, 2, 6, 8, 10, 0, 4, 12, 14,
0, 6, 8, 10, 2, 4, 12, 14, 6, 8, 10, 0, 2, 4, 12, 14,
0, 2, 4, 8, 10, 6, 12, 14, 2, 4, 8, 10, 0, 6, 12, 14,
0, 4, 8, 10, 2, 6, 12, 14, 4, 8, 10, 0, 2, 6, 12, 14,
0, 2, 8, 10, 4, 6, 12, 14, 2, 8, 10, 0, 4, 6, 12, 14,
0, 8, 10, 2, 4, 6, 12, 14, 8, 10, 0, 2, 4, 6, 12, 14,
0, 2, 4, 6, 10, 8, 12, 14, 2, 4, 6, 10, 0, 8, 12, 14,
0, 4, 6, 10, 2, 8, 12, 14, 4, 6, 10, 0, 2, 8, 12, 14,
0, 2, 6, 10, 4, 8, 12, 14, 2, 6, 10, 0, 4, 8, 12, 14,
0, 6, 10, 2, 4, 8, 12, 14, 6, 10, 0, 2, 4, 8, 12, 14,
0, 2, 4, 10, 6, 8, 12, 14, 2, 4, 10, 0, 6, 8, 12, 14,
0, 4, 10, 2, 6, 8, 12, 14, 4, 10, 0, 2, 6, 8, 12, 14,
0, 2, 10, 4, 6, 8, 12, 14, 2, 10, 0, 4, 6, 8, 12, 14,
0, 10, 2, 4, 6, 8, 12, 14, 10, 0, 2, 4, 6, 8, 12, 14,
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 6, 8, 0, 10, 12, 14,
0, 4, 6, 8, 2, 10, 12, 14, 4, 6, 8, 0, 2, 10, 12, 14,
0, 2, 6, 8, 4, 10, 12, 14, 2, 6, 8, 0, 4, 10, 12, 14,
0, 6, 8, 2, 4, 10, 12, 14, 6, 8, 0, 2, 4, 10, 12, 14,
0, 2, 4, 8, 6, 10, 12, 14, 2, 4, 8, 0, 6, 10, 12, 14,
0, 4, 8, 2, 6, 10, 12, 14, 4, 8, 0, 2, 6, 10, 12, 14,
0, 2, 8, 4, 6, 10, 12, 14, 2, 8, 0, 4, 6, 10, 12, 14,
0, 8, 2, 4, 6, 10, 12, 14, 8, 0, 2, 4, 6, 10, 12, 14,
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 6, 0, 8, 10, 12, 14,
0, 4, 6, 2, 8, 10, 12, 14, 4, 6, 0, 2, 8, 10, 12, 14,
0, 2, 6, 4, 8, 10, 12, 14, 2, 6, 0, 4, 8, 10, 12, 14,
0, 6, 2, 4, 8, 10, 12, 14, 6, 0, 2, 4, 8, 10, 12, 14,
0, 2, 4, 6, 8, 10, 12, 14, 2, 4, 0, 6, 8, 10, 12, 14,
0, 4, 2, 6, 8, 10, 12, 14, 4, 0, 2, 6, 8, 10, 12, 14,
0, 2, 4, 6, 8, 10, 12, 14, 2, 0, 4, 6, 8, 10, 12, 14,
0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14};
const VFromD<decltype(d8t)> byte_idx{Load(d8, table + mask_bits * 8).raw};
const VFromD<decltype(du)> pairs = ZipLower(byte_idx, byte_idx);
return BitCast(d, pairs + Set(du, 0x0100));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromBits128(D d, uint64_t mask_bits) {
do { } while (0);
alignas(16) static constexpr uint8_t u8_indices[256] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
8, 9, 10, 11, 0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15,
0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15,
4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 12, 13, 14, 15,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7, 8, 9, 10, 11,
4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 8, 9, 10, 11,
0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 8, 9, 10, 11,
8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15, 4, 5, 6, 7,
4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, Load(d8, u8_indices + 16 * mask_bits));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromNotBits128(D d, uint64_t mask_bits) {
do { } while (0);
alignas(16) static constexpr uint8_t u8_indices[256] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 5,
6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3,
8, 9, 10, 11, 12, 13, 14, 15, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15, 0, 1,
2, 3, 8, 9, 10, 11, 0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7,
8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
10, 11, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 12, 13, 14, 15, 0, 1,
2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15, 8, 9, 10, 11,
0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5,
6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 5, 6, 7, 0, 1, 2, 3,
8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
12, 13, 14, 15};
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, Load(d8, u8_indices + 16 * mask_bits));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromBits128(D d, uint64_t mask_bits) {
do { } while (0);
alignas(16) static constexpr uint8_t u8_indices[64] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, Load(d8, u8_indices + 16 * mask_bits));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> IndicesFromNotBits128(D d, uint64_t mask_bits) {
do { } while (0);
alignas(16) static constexpr uint8_t u8_indices[64] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, Load(d8, u8_indices + 16 * mask_bits));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> CompressBits(Vec128<T, N> v, uint64_t mask_bits) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
do { } while (0);
const auto indices = BitCast(du, detail::IndicesFromBits128(d, mask_bits));
return BitCast(d, TableLookupBytes(BitCast(du, v), indices));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> CompressNotBits(Vec128<T, N> v, uint64_t mask_bits) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
do { } while (0);
const auto indices = BitCast(du, detail::IndicesFromNotBits128(d, mask_bits));
return BitCast(d, TableLookupBytes(BitCast(du, v), indices));
}
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> Compress(Vec128<T, 1> v, Mask128<T, 1> ) {
return v;
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Compress(Vec128<T> v, Mask128<T> mask) {
const DFromV<decltype(v)> d;
const Vec128<T> m = VecFromMask(d, mask);
const Vec128<T> maskL = DupEven(m);
const Vec128<T> maskH = DupOdd(m);
const Vec128<T> swap = AndNot(maskL, maskH);
return IfVecThenElse(swap, Shuffle01(v), v);
}
template <typename T, size_t N, hwy::EnableIf<((size_t{1} << sizeof(T)) & ((1 << 2) | (1 << 4))) != 0>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Compress(Vec128<T, N> v, Mask128<T, N> mask) {
return detail::CompressBits(v, detail::BitsFromMask(mask));
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> CompressNot(Vec128<T, 1> v, Mask128<T, 1> ) {
return v;
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> CompressNot(Vec128<T> v, Mask128<T> mask) {
const DFromV<decltype(v)> d;
const Vec128<T> m = VecFromMask(d, mask);
const Vec128<T> maskL = DupEven(m);
const Vec128<T> maskH = DupOdd(m);
const Vec128<T> swap = AndNot(maskH, maskL);
return IfVecThenElse(swap, Shuffle01(v), v);
}
template <typename T, size_t N, hwy::EnableIf<((size_t{1} << sizeof(T)) & ((1 << 2) | (1 << 4))) != 0>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> CompressNot(Vec128<T, N> v, Mask128<T, N> mask) {
if (N < 16 / sizeof(T)) {
return detail::CompressBits(v, detail::BitsFromMask(Not(mask)));
}
return detail::CompressNotBits(v, detail::BitsFromMask(mask));
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t> CompressBlocksNot(Vec128<uint64_t> v,
Mask128<uint64_t> ) {
return v;
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> CompressBits(Vec128<T, N> v,
const uint8_t* __restrict__ bits) {
uint64_t mask_bits = 0;
constexpr size_t kNumBytes = (N + 7) / 8;
CopyBytes<kNumBytes>(bits, &mask_bits);
if (N < 8) {
mask_bits &= (1ull << N) - 1;
}
return detail::CompressBits(v, mask_bits);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressStore(VFromD<D> v, MFromD<D> m, D d,
TFromD<D>* __restrict__ unaligned) {
const RebindToUnsigned<decltype(d)> du;
const uint64_t mask_bits = detail::BitsFromMask(m);
do { } while (0);
const size_t count = PopCount(mask_bits);
const auto indices = BitCast(du, detail::IndicesFromBits128(d, mask_bits));
const auto compressed = BitCast(d, TableLookupBytes(BitCast(du, v), indices));
StoreU(compressed, d, unaligned);
detail::MaybeUnpoison(unaligned, count);
return count;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressBlendedStore(VFromD<D> v, MFromD<D> m, D d,
TFromD<D>* __restrict__ unaligned) {
const RebindToUnsigned<decltype(d)> du;
const uint64_t mask_bits = detail::BitsFromMask(m);
do { } while (0);
const size_t count = PopCount(mask_bits);
const auto indices = BitCast(du, detail::IndicesFromBits128(d, mask_bits));
const auto compressed = BitCast(d, TableLookupBytes(BitCast(du, v), indices));
BlendedStore(compressed, FirstN(d, count), d, unaligned);
detail::MaybeUnpoison(unaligned, count);
return count;
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressBitsStore(VFromD<D> v, const uint8_t* __restrict__ bits,
D d, TFromD<D>* __restrict__ unaligned) {
const RebindToUnsigned<decltype(d)> du;
uint64_t mask_bits = 0;
constexpr size_t kN = MaxLanes(d);
constexpr size_t kNumBytes = (kN + 7) / 8;
CopyBytes<kNumBytes>(bits, &mask_bits);
if (kN < 8) {
mask_bits &= (1ull << kN) - 1;
}
const size_t count = PopCount(mask_bits);
const auto indices = BitCast(du, detail::IndicesFromBits128(d, mask_bits));
const auto compressed = BitCast(d, TableLookupBytes(BitCast(du, v), indices));
StoreU(compressed, d, unaligned);
detail::MaybeUnpoison(unaligned, count);
return count;
}
template <class T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, 1> SetAtOrAfterFirst(Mask128<T, 1> mask) {
return mask;
}
template <class T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, 2> SetAtOrAfterFirst(Mask128<T, 2> mask) {
const FixedTag<T, 2> d;
const auto vmask = VecFromMask(d, mask);
return MaskFromVec(Or(vmask, InterleaveLower(vmask, vmask)));
}
template <class T, size_t N, hwy::EnableIf<(N > 2)>* = nullptr, hwy::EnableIf<N * sizeof(T) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> SetAtOrAfterFirst(Mask128<T, N> mask) {
const Simd<T, N, 0> d;
const auto vmask = VecFromMask(d, mask);
const auto neg_vmask =
ResizeBitCast(d, Neg(ResizeBitCast(Full64<int64_t>(), vmask)));
return MaskFromVec(Or(vmask, neg_vmask));
}
template <class T, hwy::EnableIf<sizeof(T) != (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T> SetAtOrAfterFirst(Mask128<T> mask) {
const Full128<T> d;
const Repartition<int64_t, decltype(d)> di64;
const Repartition<float, decltype(d)> df32;
const Repartition<int32_t, decltype(d)> di32;
using VF = VFromD<decltype(df32)>;
auto vmask = BitCast(di64, VecFromMask(d, mask));
vmask = Or(vmask, Neg(vmask));
const auto vmask2 = BroadcastSignBit(
BitCast(di32, VF{_mm_shuffle_ps(Zero(df32).raw, BitCast(df32, vmask).raw,
(((
1
) << 6) | ((
1
) << 4) | ((
0
) << 2) | (
0
))
)}));
return MaskFromVec(BitCast(d, Or(vmask, BitCast(di64, vmask2))));
}
template <class T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> SetBeforeFirst(Mask128<T, N> mask) {
return Not(SetAtOrAfterFirst(mask));
}
template <class T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, 1> SetOnlyFirst(Mask128<T, 1> mask) {
return mask;
}
template <class T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, 2> SetOnlyFirst(Mask128<T, 2> mask) {
const FixedTag<T, 2> d;
const RebindToSigned<decltype(d)> di;
const auto vmask = BitCast(di, VecFromMask(d, mask));
const auto zero = Zero(di);
const auto vmask2 = VecFromMask(di, InterleaveLower(zero, vmask) == zero);
return MaskFromVec(BitCast(d, And(vmask, vmask2)));
}
template <class T, size_t N, hwy::EnableIf<(N > 2)>* = nullptr, hwy::EnableIf<N * sizeof(T) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> SetOnlyFirst(Mask128<T, N> mask) {
const Simd<T, N, 0> d;
const RebindToSigned<decltype(d)> di;
const auto vmask = ResizeBitCast(Full64<int64_t>(), VecFromMask(d, mask));
const auto only_first_vmask =
BitCast(d, Neg(ResizeBitCast(di, And(vmask, Neg(vmask)))));
return MaskFromVec(only_first_vmask);
}
template <class T, hwy::EnableIf<sizeof(T) != (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T> SetOnlyFirst(Mask128<T> mask) {
const Full128<T> d;
const RebindToSigned<decltype(d)> di;
const Repartition<int64_t, decltype(d)> di64;
const auto zero = Zero(di64);
const auto vmask = BitCast(di64, VecFromMask(d, mask));
const auto vmask2 = VecFromMask(di64, InterleaveLower(zero, vmask) == zero);
const auto only_first_vmask = Neg(BitCast(di, And(vmask, Neg(vmask))));
return MaskFromVec(BitCast(d, And(only_first_vmask, BitCast(di, vmask2))));
}
template <class T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, 1> SetAtOrBeforeFirst(Mask128<T, 1> ) {
const FixedTag<T, 1> d;
const RebindToSigned<decltype(d)> di;
using TI = MakeSigned<T>;
return RebindMask(d, MaskFromVec(Set(di, TI(-1))));
}
template <class T, size_t N, hwy::EnableIf<(N > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> SetAtOrBeforeFirst(Mask128<T, N> mask) {
const Simd<T, N, 0> d;
return SetBeforeFirst(MaskFromVec(ShiftLeftLanes<1>(VecFromMask(d, mask))));
}
namespace detail {
template <typename T>
inline __attribute__((always_inline)) Vec128<T, 1> SumOfLanes(Vec128<T, 1> v) {
return v;
}
template <typename T>
inline __attribute__((always_inline)) T ReduceSum(Vec128<T, 1> v) {
return GetLane(v);
}
template <typename T>
inline __attribute__((always_inline)) Vec128<T, 1> MinOfLanes(Vec128<T, 1> v) {
return v;
}
template <typename T>
inline __attribute__((always_inline)) Vec128<T, 1> MaxOfLanes(Vec128<T, 1> v) {
return v;
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, 2> SumOfLanes(Vec128<T, 2> v10) {
return v10 + Shuffle2301(v10);
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) T ReduceSum(Vec128<T, 2> v10) {
return GetLane(SumOfLanes(v10));
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, 2> MinOfLanes(Vec128<T, 2> v10) {
return Min(v10, Shuffle2301(v10));
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T, 2> MaxOfLanes(Vec128<T, 2> v10) {
return Max(v10, Shuffle2301(v10));
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> SumOfLanes(Vec128<T> v3210) {
const Vec128<T> v1032 = Shuffle1032(v3210);
const Vec128<T> v31_20_31_20 = v3210 + v1032;
const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20);
return v20_31_20_31 + v31_20_31_20;
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) T ReduceSum(Vec128<T> v3210) {
return GetLane(SumOfLanes(v3210));
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> MinOfLanes(Vec128<T> v3210) {
const Vec128<T> v1032 = Shuffle1032(v3210);
const Vec128<T> v31_20_31_20 = Min(v3210, v1032);
const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20);
return Min(v20_31_20_31, v31_20_31_20);
}
template <typename T, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> MaxOfLanes(Vec128<T> v3210) {
const Vec128<T> v1032 = Shuffle1032(v3210);
const Vec128<T> v31_20_31_20 = Max(v3210, v1032);
const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20);
return Max(v20_31_20_31, v31_20_31_20);
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> SumOfLanes(Vec128<T> v10) {
const Vec128<T> v01 = Shuffle01(v10);
return v10 + v01;
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) T ReduceSum(Vec128<T> v10) {
return GetLane(SumOfLanes(v10));
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> MinOfLanes(Vec128<T> v10) {
const Vec128<T> v01 = Shuffle01(v10);
return Min(v10, v01);
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
inline __attribute__((always_inline)) Vec128<T> MaxOfLanes(Vec128<T> v10) {
const Vec128<T> v01 = Shuffle01(v10);
return Max(v10, v01);
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) uint16_t ReduceSum(Vec128<uint16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto sum = ReduceSum(even + odd);
return static_cast<uint16_t>(sum);
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<uint16_t, N> SumOfLanes(Vec128<uint16_t, N> v) {
const DFromV<decltype(v)> d;
return Set(d, ReduceSum(v));
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) int16_t ReduceSum(Vec128<int16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = ShiftRight<16>(ShiftLeft<16>(BitCast(d32, v)));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto sum = ReduceSum(even + odd);
return static_cast<int16_t>(sum);
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int16_t, N> SumOfLanes(Vec128<int16_t, N> v) {
const DFromV<decltype(v)> d;
return Set(d, ReduceSum(v));
}
inline __attribute__((always_inline)) uint8_t ReduceSum(Vec64<uint8_t> v) {
return static_cast<uint8_t>(GetLane(SumsOf8(v)) & 0xFF);
}
inline __attribute__((always_inline)) Vec64<uint8_t> SumOfLanes(Vec64<uint8_t> v) {
const Full64<uint8_t> d;
return Set(d, ReduceSum(v));
}
inline __attribute__((always_inline)) uint8_t ReduceSum(Vec128<uint8_t> v) {
uint64_t sums = ReduceSum(SumsOf8(v));
return static_cast<uint8_t>(sums & 0xFF);
}
inline __attribute__((always_inline)) Vec128<uint8_t> SumOfLanes(Vec128<uint8_t> v) {
const DFromV<decltype(v)> d;
return Set(d, ReduceSum(v));
}
template <size_t N, hwy::EnableIf<(N * sizeof(int8_t) > 4)>* = nullptr>
inline __attribute__((always_inline)) int8_t ReduceSum(const Vec128<int8_t, N> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const auto is_neg = v < Zero(d);
const auto positive = SumsOf8(BitCast(du, IfThenZeroElse(is_neg, v)));
const auto negative = SumsOf8(BitCast(du, IfThenElseZero(is_neg, Abs(v))));
return static_cast<int8_t>(ReduceSum(positive - negative) & 0xFF);
}
template <size_t N, hwy::EnableIf<(N * sizeof(int8_t) > 4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int8_t, N> SumOfLanes(const Vec128<int8_t, N> v) {
const DFromV<decltype(v)> d;
return Set(d, ReduceSum(v));
}
inline __attribute__((always_inline)) Vec128<uint16_t> MinOfLanes(Vec128<uint16_t> v) {
using V = decltype(v);
return Broadcast<0>(V{_mm_minpos_epu16(v.raw)});
}
inline __attribute__((always_inline)) Vec64<uint8_t> MinOfLanes(Vec64<uint8_t> v) {
const DFromV<decltype(v)> d;
const Rebind<uint16_t, decltype(d)> d16;
return TruncateTo(d, MinOfLanes(PromoteTo(d16, v)));
}
inline __attribute__((always_inline)) Vec128<uint8_t> MinOfLanes(Vec128<uint8_t> v) {
const Half<DFromV<decltype(v)>> d;
Vec64<uint8_t> result =
Min(MinOfLanes(UpperHalf(d, v)), MinOfLanes(LowerHalf(d, v)));
return Combine(DFromV<decltype(v)>(), result, result);
}
inline __attribute__((always_inline)) Vec128<uint16_t> MaxOfLanes(Vec128<uint16_t> v) {
const Vec128<uint16_t> m(Set(DFromV<decltype(v)>(), LimitsMax<uint16_t>()));
return m - MinOfLanes(m - v);
}
inline __attribute__((always_inline)) Vec64<uint8_t> MaxOfLanes(Vec64<uint8_t> v) {
const Vec64<uint8_t> m(Set(DFromV<decltype(v)>(), LimitsMax<uint8_t>()));
return m - MinOfLanes(m - v);
}
inline __attribute__((always_inline)) Vec128<uint8_t> MaxOfLanes(Vec128<uint8_t> v) {
const Vec128<uint8_t> m(Set(DFromV<decltype(v)>(), LimitsMax<uint8_t>()));
return m - MinOfLanes(m - v);
}
template <size_t N, hwy::EnableIf<(N * sizeof(int8_t) > 4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int8_t, N> MinOfLanes(Vec128<int8_t, N> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const auto mask = SignBit(du);
const auto vu = Xor(BitCast(du, v), mask);
return BitCast(d, Xor(MinOfLanes(vu), mask));
}
template <size_t N, hwy::EnableIf<(N * sizeof(int8_t) > 4)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int8_t, N> MaxOfLanes(Vec128<int8_t, N> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const auto mask = SignBit(du);
const auto vu = Xor(BitCast(du, v), mask);
return BitCast(d, Xor(MaxOfLanes(vu), mask));
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<uint16_t, N> MinOfLanes(Vec128<uint16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto min = MinOfLanes(Min(even, odd));
return OddEven(BitCast(d, ShiftLeft<16>(min)), BitCast(d, min));
}
template <size_t N, hwy::EnableIf<(N * sizeof(int16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int16_t, N> MinOfLanes(Vec128<int16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = ShiftRight<16>(ShiftLeft<16>(BitCast(d32, v)));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto min = MinOfLanes(Min(even, odd));
return OddEven(BitCast(d, ShiftLeft<16>(min)), BitCast(d, min));
}
template <size_t N, hwy::EnableIf<(N * sizeof(uint16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<uint16_t, N> MaxOfLanes(Vec128<uint16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto min = MaxOfLanes(Max(even, odd));
return OddEven(BitCast(d, ShiftLeft<16>(min)), BitCast(d, min));
}
template <size_t N, hwy::EnableIf<(N * sizeof(int16_t) > 2)>* = nullptr>
inline __attribute__((always_inline)) Vec128<int16_t, N> MaxOfLanes(Vec128<int16_t, N> v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> d32;
const auto even = ShiftRight<16>(ShiftLeft<16>(BitCast(d32, v)));
const auto odd = ShiftRight<16>(BitCast(d32, v));
const auto min = MaxOfLanes(Max(even, odd));
return OddEven(BitCast(d, ShiftLeft<16>(min)), BitCast(d, min));
}
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SumOfLanes(D , VFromD<D> v) {
return detail::SumOfLanes(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) TFromD<D> ReduceSum(D , VFromD<D> v) {
return detail::ReduceSum(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> MinOfLanes(D , VFromD<D> v) {
return detail::MinOfLanes(v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> MaxOfLanes(D , VFromD<D> v) {
return detail::MaxOfLanes(v);
}
namespace detail {
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Lt128Vec(const D d, const V a, const V b) {
static_assert(IsSame<TFromD<D>, uint64_t>(), "D must be u64");
const auto eqHL = Eq(a, b);
const V ltHL = VecFromMask(d, Lt(a, b));
const V ltLX = ShiftLeftLanes<1>(ltHL);
const V vecHx = IfThenElse(eqHL, ltLX, ltHL);
return InterleaveUpper(d, vecHx, vecHx);
}
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Eq128Vec(const D d, const V a, const V b) {
static_assert(IsSame<TFromD<D>, uint64_t>(), "D must be u64");
const auto eqHL = VecFromMask(d, Eq(a, b));
const auto eqLH = Reverse2(d, eqHL);
return And(eqHL, eqLH);
}
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Ne128Vec(const D d, const V a, const V b) {
static_assert(IsSame<TFromD<D>, uint64_t>(), "D must be u64");
const auto neHL = VecFromMask(d, Ne(a, b));
const auto neLH = Reverse2(d, neHL);
return Or(neHL, neLH);
}
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Lt128UpperVec(const D d, const V a, const V b) {
const V ltHL = VecFromMask(d, Lt(a, b));
return InterleaveUpper(d, ltHL, ltHL);
}
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Eq128UpperVec(const D d, const V a, const V b) {
const V eqHL = VecFromMask(d, Eq(a, b));
return InterleaveUpper(d, eqHL, eqHL);
}
template <class D, class V = VFromD<D>>
inline __attribute__((always_inline)) V Ne128UpperVec(const D d, const V a, const V b) {
const V neHL = VecFromMask(d, Ne(a, b));
return InterleaveUpper(d, neHL, neHL);
}
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Lt128(D d, const V a, const V b) {
return MaskFromVec(detail::Lt128Vec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Eq128(D d, const V a, const V b) {
return MaskFromVec(detail::Eq128Vec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Ne128(D d, const V a, const V b) {
return MaskFromVec(detail::Ne128Vec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Lt128Upper(D d, const V a, const V b) {
return MaskFromVec(detail::Lt128UpperVec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Eq128Upper(D d, const V a, const V b) {
return MaskFromVec(detail::Eq128UpperVec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Ne128Upper(D d, const V a, const V b) {
return MaskFromVec(detail::Ne128UpperVec(d, a, b));
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Min128(D d, const V a, const V b) {
return IfVecThenElse(detail::Lt128Vec(d, a, b), a, b);
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Max128(D d, const V a, const V b) {
return IfVecThenElse(detail::Lt128Vec(d, b, a), a, b);
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Min128Upper(D d, const V a, const V b) {
return IfVecThenElse(detail::Lt128UpperVec(d, a, b), a, b);
}
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Max128Upper(D d, const V a, const V b) {
return IfVecThenElse(detail::Lt128UpperVec(d, b, a), a, b);
}
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC diagnostic pop
#pragma GCC push_options
#pragma GCC target "sse2,ssse3" ",sse4.1,sse4.2" ",pclmul,aes"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSE4 {
template <class V>
using LaneType = decltype(GetLane(V()));
template <class D>
using Vec = decltype(Zero(D()));
template <class D>
using Mask = decltype(MaskFromVec(Zero(D())));
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Clamp(const V v, const V lo, const V hi) {
return Min(Max(lo, v), hi);
}
template <size_t kLanes, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> CombineShiftRightLanes(D d, VFromD<D> hi, VFromD<D> lo) {
constexpr size_t kBytes = kLanes * sizeof(TFromD<D>);
static_assert(kBytes < 16, "Shift count is per-block");
return CombineShiftRightBytes<kBytes>(d, hi, lo);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec<D> SignBit(D d) {
const RebindToUnsigned<decltype(d)> du;
return BitCast(d, Set(du, SignMask<TFromD<D>>()));
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec<D> NaN(D d) {
const RebindToSigned<D> di;
return BitCast(d, Set(di, LimitsMax<TFromD<decltype(di)>>()));
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec<D> Inf(D d) {
const RebindToUnsigned<D> du;
using T = TFromD<D>;
using TU = TFromD<decltype(du)>;
const TU max_x2 = static_cast<TU>(MaxExponentTimes2<T>());
return BitCast(d, Set(du, max_x2 >> 1));
}
namespace detail {
template <size_t kFromVectSize, size_t kToVectSize, class DTo, class DFrom,
hwy::EnableIf<(kToVectSize <= kFromVectSize)>* = nullptr>
inline __attribute__((always_inline)) VFromD<DTo> ZeroExtendResizeBitCast(
hwy::SizeTag<kFromVectSize> ,
hwy::SizeTag<kToVectSize> , DTo d_to, DFrom ,
VFromD<DFrom> v) {
return ResizeBitCast(d_to, v);
}
template <size_t kFromVectSize, size_t kToVectSize, class DTo, class DFrom,
hwy::EnableIf<(kToVectSize == kFromVectSize * 2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<DTo> ZeroExtendResizeBitCast(
hwy::SizeTag<kFromVectSize> ,
hwy::SizeTag<kToVectSize> , DTo d_to, DFrom d_from,
VFromD<DFrom> v) {
const Twice<decltype(d_from)> dt_from;
return BitCast(d_to, ZeroExtendVector(dt_from, v));
}
template <size_t kFromVectSize, size_t kToVectSize, class DTo, class DFrom,
hwy::EnableIf<(kToVectSize > kFromVectSize * 2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<DTo> ZeroExtendResizeBitCast(
hwy::SizeTag<kFromVectSize> ,
hwy::SizeTag<kToVectSize> , DTo d_to, DFrom ,
VFromD<DFrom> v) {
using TFrom = TFromD<DFrom>;
constexpr size_t kNumOfFromLanes = kFromVectSize / sizeof(TFrom);
const Repartition<TFrom, decltype(d_to)> d_resize_to;
return BitCast(d_to, IfThenElseZero(FirstN(d_resize_to, kNumOfFromLanes),
ResizeBitCast(d_resize_to, v)));
}
}
template <class DTo, class DFrom>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DTo> ZeroExtendResizeBitCast(DTo d_to, DFrom d_from,
VFromD<DFrom> v) {
return detail::ZeroExtendResizeBitCast(hwy::SizeTag<d_from.MaxBytes()>(),
hwy::SizeTag<d_to.MaxBytes()>(), d_to,
d_from, v);
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void SafeFillN(const size_t num, const T value, D d,
T* __restrict__ to) {
(void)d;
for (size_t i = 0; i < num; ++i) {
to[i] = value;
}
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void SafeCopyN(const size_t num, D d, const T* __restrict__ from,
T* __restrict__ to) {
(void)d;
for (size_t i = 0; i < num; ++i) {
to[i] = from[i];
}
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V BitwiseIfThenElse(V mask, V yes, V no) {
return Or(And(mask, yes), AndNot(mask, no));
}
template <class D, hwy::EnableIf<(D::kPrivateLanes > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved2(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1) {
const VFromD<D> A = LoadU(d, unaligned);
const VFromD<D> B = LoadU(d, unaligned + Lanes(d));
v0 = ConcatEven(d, B, A);
v1 = ConcatOdd(d, B, A);
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved2(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1) {
v0 = LoadU(d, unaligned + 0);
v1 = LoadU(d, unaligned + 1);
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) void LoadTransposedBlocks3(D d,
const TFromD<D>* __restrict__ unaligned,
VFromD<D>& A, VFromD<D>& B,
VFromD<D>& C) {
constexpr size_t kN = MaxLanes(d);
A = LoadU(d, unaligned + 0 * kN);
B = LoadU(d, unaligned + 1 * kN);
C = LoadU(d, unaligned + 2 * kN);
}
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (16)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
const RebindToUnsigned<decltype(d)> du;
using V = VFromD<D>;
V A;
V B;
V C;
detail::LoadTransposedBlocks3(d, unaligned, A, B, C);
constexpr uint8_t Z = 0x80;
alignas(16) static constexpr uint8_t kIdx_v0A[16] = {
0, 3, 6, 9, 12, 15, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0B[16] = {
Z, Z, Z, Z, Z, Z, 2, 5, 8, 11, 14, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 1, 4, 7, 10, 13};
alignas(16) static constexpr uint8_t kIdx_v1A[16] = {
1, 4, 7, 10, 13, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1B[16] = {
Z, Z, Z, Z, Z, 0, 3, 6, 9, 12, 15, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 2, 5, 8, 11, 14};
alignas(16) static constexpr uint8_t kIdx_v2A[16] = {
2, 5, 8, 11, 14, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2B[16] = {
Z, Z, Z, Z, Z, 1, 4, 7, 10, 13, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 0, 3, 6, 9, 12, 15};
const V v0L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v0A)));
const V v0M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v0B)));
const V v0U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v0C)));
const V v1L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v1A)));
const V v1M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v1B)));
const V v1U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v1C)));
const V v2L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v2A)));
const V v2M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v2B)));
const V v2U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v2C)));
v0 = Xor3(v0L, v0M, v0U);
v1 = Xor3(v1L, v1M, v1U);
v2 = Xor3(v2L, v2M, v2U);
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (8)>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
const RebindToUnsigned<decltype(d)> du;
using V = VFromD<D>;
V A;
V B;
V C;
detail::LoadTransposedBlocks3(d, unaligned, A, B, C);
constexpr uint8_t Z = 0x80;
alignas(16) static constexpr uint8_t kIdx_v0A[16] = {0, 3, 6, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0B[16] = {Z, Z, Z, 1, 4, 7, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0C[16] = {Z, Z, Z, Z, Z, Z, 2, 5};
alignas(16) static constexpr uint8_t kIdx_v1A[16] = {1, 4, 7, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1B[16] = {Z, Z, Z, 2, 5, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1C[16] = {Z, Z, Z, Z, Z, 0, 3, 6};
alignas(16) static constexpr uint8_t kIdx_v2A[16] = {2, 5, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2B[16] = {Z, Z, 0, 3, 6, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2C[16] = {Z, Z, Z, Z, Z, 1, 4, 7};
const V v0L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v0A)));
const V v0M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v0B)));
const V v0U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v0C)));
const V v1L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v1A)));
const V v1M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v1B)));
const V v1U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v1C)));
const V v2L = BitCast(d, TableLookupBytesOr0(A, LoadDup128(du, kIdx_v2A)));
const V v2M = BitCast(d, TableLookupBytesOr0(B, LoadDup128(du, kIdx_v2B)));
const V v2U = BitCast(d, TableLookupBytesOr0(C, LoadDup128(du, kIdx_v2C)));
v0 = Xor3(v0L, v0M, v0U);
v1 = Xor3(v1L, v1M, v1U);
v2 = Xor3(v2L, v2M, v2U);
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (8)>* = nullptr, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
const RebindToUnsigned<decltype(d)> du;
const Repartition<uint8_t, decltype(du)> du8;
using V = VFromD<D>;
V A;
V B;
V C;
detail::LoadTransposedBlocks3(d, unaligned, A, B, C);
constexpr uint8_t Z = 0x80;
alignas(16) static constexpr uint8_t kIdx_v0A[16] = {
0x00, 0x01, 0x06, 0x07, 0x0C, 0x0D, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0B[16] = {
Z, Z, Z, Z, Z, Z, 0x02, 0x03, 0x08, 0x09, 0x0E, 0x0F, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v0C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 0x04, 0x05, 0x0A, 0x0B};
alignas(16) static constexpr uint8_t kIdx_v1A[16] = {
0x02, 0x03, 0x08, 0x09, 0x0E, 0x0F, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1B[16] = {
Z, Z, Z, Z, Z, Z, 0x04, 0x05, 0x0A, 0x0B, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v1C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 0x00, 0x01, 0x06, 0x07, 0x0C, 0x0D};
alignas(16) static constexpr uint8_t kIdx_v2A[16] = {
0x04, 0x05, 0x0A, 0x0B, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2B[16] = {
Z, Z, Z, Z, 0x00, 0x01, 0x06, 0x07, 0x0C, 0x0D, Z, Z, Z, Z, Z, Z};
alignas(16) static constexpr uint8_t kIdx_v2C[16] = {
Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, 0x02, 0x03, 0x08, 0x09, 0x0E, 0x0F};
const V v0L = TableLookupBytesOr0(A, BitCast(d, LoadDup128(du8, kIdx_v0A)));
const V v0M = TableLookupBytesOr0(B, BitCast(d, LoadDup128(du8, kIdx_v0B)));
const V v0U = TableLookupBytesOr0(C, BitCast(d, LoadDup128(du8, kIdx_v0C)));
const V v1L = TableLookupBytesOr0(A, BitCast(d, LoadDup128(du8, kIdx_v1A)));
const V v1M = TableLookupBytesOr0(B, BitCast(d, LoadDup128(du8, kIdx_v1B)));
const V v1U = TableLookupBytesOr0(C, BitCast(d, LoadDup128(du8, kIdx_v1C)));
const V v2L = TableLookupBytesOr0(A, BitCast(d, LoadDup128(du8, kIdx_v2A)));
const V v2M = TableLookupBytesOr0(B, BitCast(d, LoadDup128(du8, kIdx_v2B)));
const V v2U = TableLookupBytesOr0(C, BitCast(d, LoadDup128(du8, kIdx_v2C)));
v0 = Xor3(v0L, v0M, v0U);
v1 = Xor3(v1L, v1M, v1U);
v2 = Xor3(v2L, v2M, v2U);
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
using V = VFromD<D>;
V A;
V B;
V C;
detail::LoadTransposedBlocks3(d, unaligned, A, B, C);
const V vxx_02_03_xx = OddEven(C, B);
v0 = detail::ShuffleTwo1230(A, vxx_02_03_xx);
const V vxx_xx_10_11 = OddEven(A, B);
const V v12_13_xx_xx = OddEven(B, C);
v1 = detail::ShuffleTwo2301(vxx_xx_10_11, v12_13_xx_xx);
const V vxx_20_21_xx = OddEven(B, A);
v2 = detail::ShuffleTwo3012(vxx_20_21_xx, C);
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
VFromD<D> A;
VFromD<D> B;
VFromD<D> C;
detail::LoadTransposedBlocks3(d, unaligned, A, B, C);
v0 = OddEven(B, A);
v1 = CombineShiftRightBytes<sizeof(TFromD<D>)>(d, C, A);
v2 = OddEven(C, B);
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const T* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
v0 = LoadU(d, unaligned + 0);
v1 = LoadU(d, unaligned + 1);
v2 = LoadU(d, unaligned + 2);
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) void LoadTransposedBlocks4(D d,
const TFromD<D>* __restrict__ unaligned,
VFromD<D>& vA, VFromD<D>& vB,
VFromD<D>& vC, VFromD<D>& vD) {
constexpr size_t kN = MaxLanes(d);
vA = LoadU(d, unaligned + 0 * kN);
vB = LoadU(d, unaligned + 1 * kN);
vC = LoadU(d, unaligned + 2 * kN);
vD = LoadU(d, unaligned + 3 * kN);
}
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (16)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved4(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2,
VFromD<D>& v3) {
const Repartition<uint64_t, decltype(d)> d64;
using V64 = VFromD<decltype(d64)>;
using V = VFromD<D>;
V vA;
V vB;
V vC;
V vD;
detail::LoadTransposedBlocks4(d, unaligned, vA, vB, vC, vD);
const V v5140 = InterleaveLower(d, vA, vB);
const V vd9c8 = InterleaveLower(d, vC, vD);
const V v7362 = InterleaveUpper(d, vA, vB);
const V vfbea = InterleaveUpper(d, vC, vD);
const V v6420 = InterleaveLower(d, v5140, v7362);
const V veca8 = InterleaveLower(d, vd9c8, vfbea);
const V v7531 = InterleaveUpper(d, v5140, v7362);
const V vfdb9 = InterleaveUpper(d, vd9c8, vfbea);
const V64 v10L = BitCast(d64, InterleaveLower(d, v6420, v7531));
const V64 v10U = BitCast(d64, InterleaveLower(d, veca8, vfdb9));
const V64 v32L = BitCast(d64, InterleaveUpper(d, v6420, v7531));
const V64 v32U = BitCast(d64, InterleaveUpper(d, veca8, vfdb9));
v0 = BitCast(d, InterleaveLower(d64, v10L, v10U));
v1 = BitCast(d, InterleaveUpper(d64, v10L, v10U));
v2 = BitCast(d, InterleaveLower(d64, v32L, v32U));
v3 = BitCast(d, InterleaveUpper(d64, v32L, v32U));
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved4(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2,
VFromD<D>& v3) {
using TW = hwy::UnsignedFromSize<d.MaxBytes() == 8 ? 4 : 8>;
const Repartition<TW, decltype(d)> dw;
using VW = VFromD<decltype(dw)>;
VFromD<D> vA;
VFromD<D> vB;
VFromD<D> vC;
VFromD<D> vD;
detail::LoadTransposedBlocks4(d, unaligned, vA, vB, vC, vD);
const VFromD<D> va820 = InterleaveLower(d, vA, vB);
const VFromD<D> vec64 = InterleaveLower(d, vC, vD);
const VFromD<D> vb931 = InterleaveUpper(d, vA, vB);
const VFromD<D> vfd75 = InterleaveUpper(d, vC, vD);
const VW v10_b830 =
BitCast(dw, InterleaveLower(d, va820, vb931));
const VW v10_fc74 =
BitCast(dw, InterleaveLower(d, vec64, vfd75));
const VW v32_b830 =
BitCast(dw, InterleaveUpper(d, va820, vb931));
const VW v32_fc74 =
BitCast(dw, InterleaveUpper(d, vec64, vfd75));
v0 = BitCast(d, InterleaveLower(dw, v10_b830, v10_fc74));
v1 = BitCast(d, InterleaveUpper(dw, v10_b830, v10_fc74));
v2 = BitCast(d, InterleaveLower(dw, v32_b830, v32_fc74));
v3 = BitCast(d, InterleaveUpper(dw, v32_b830, v32_fc74));
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved4(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2,
VFromD<D>& v3) {
using V = VFromD<D>;
V vA;
V vB;
V vC;
V vD;
detail::LoadTransposedBlocks4(d, unaligned, vA, vB, vC, vD);
const V v10e = InterleaveLower(d, vA, vC);
const V v10o = InterleaveLower(d, vB, vD);
const V v32e = InterleaveUpper(d, vA, vC);
const V v32o = InterleaveUpper(d, vB, vD);
v0 = InterleaveLower(d, v10e, v10o);
v1 = InterleaveUpper(d, v10e, v10o);
v2 = InterleaveLower(d, v32e, v32o);
v3 = InterleaveUpper(d, v32e, v32o);
}
template <class D, hwy::EnableIf<((sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) < (16) ? (sizeof(TFromD<D>) * ((D::kPrivateLanes) < (16 / sizeof(TFromD<D>)) ? (D::kPrivateLanes) : (16 / sizeof(TFromD<D>)))) : (16)) / sizeof(TFromD<D>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved4(D d, const TFromD<D>* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2,
VFromD<D>& v3) {
VFromD<D> vA, vB, vC, vD;
detail::LoadTransposedBlocks4(d, unaligned, vA, vB, vC, vD);
v0 = InterleaveLower(d, vA, vC);
v1 = InterleaveUpper(d, vA, vC);
v2 = InterleaveLower(d, vB, vD);
v3 = InterleaveUpper(d, vB, vD);
}
template <class D, typename T = TFromD<D>, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved4(D d, const T* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2,
VFromD<D>& v3) {
v0 = LoadU(d, unaligned + 0);
v1 = LoadU(d, unaligned + 1);
v2 = LoadU(d, unaligned + 2);
v3 = LoadU(d, unaligned + 3);
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) void StoreTransposedBlocks2(VFromD<D> A, VFromD<D> B, D d,
TFromD<D>* __restrict__ unaligned) {
constexpr size_t kN = MaxLanes(d);
StoreU(A, d, unaligned + 0 * kN);
StoreU(B, d, unaligned + 1 * kN);
}
}
template <class D, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved2(VFromD<D> v0, VFromD<D> v1, D d,
TFromD<D>* __restrict__ unaligned) {
const auto v10L = InterleaveLower(d, v0, v1);
const auto v10U = InterleaveUpper(d, v0, v1);
detail::StoreTransposedBlocks2(v10L, v10U, d, unaligned);
}
template <class V, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved2(V part0, V part1, D d,
TFromD<D>* __restrict__ unaligned) {
const Twice<decltype(d)> d2;
const auto v0 = ZeroExtendVector(d2, part0);
const auto v1 = ZeroExtendVector(d2, part1);
const auto v10 = InterleaveLower(d2, v0, v1);
StoreU(v10, d2, unaligned);
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) void StoreTransposedBlocks3(VFromD<D> A, VFromD<D> B, VFromD<D> C,
D d, TFromD<D>* __restrict__ unaligned) {
constexpr size_t kN = MaxLanes(d);
StoreU(A, d, unaligned + 0 * kN);
StoreU(B, d, unaligned + 1 * kN);
StoreU(C, d, unaligned + 2 * kN);
}
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
const auto k5 = Set(du, TU{5});
const auto k6 = Set(du, TU{6});
alignas(16) static constexpr uint8_t tbl_v0[16] = {
0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80, 0x80,
3, 0x80, 0x80, 4, 0x80, 0x80, 5};
alignas(16) static constexpr uint8_t tbl_v1[16] = {
0x80, 0, 0x80, 0x80, 1, 0x80,
0x80, 2, 0x80, 0x80, 3, 0x80, 0x80, 4, 0x80, 0x80};
const auto shuf_A0 = LoadDup128(du, tbl_v0);
const auto shuf_A1 = LoadDup128(du, tbl_v1);
const auto shuf_A2 = CombineShiftRightBytes<15>(du, shuf_A1, shuf_A1);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const VFromD<D> A = BitCast(d, A0 | A1 | A2);
const auto shuf_B0 = shuf_A2 + k6;
const auto shuf_B1 = shuf_A0 + k5;
const auto shuf_B2 = shuf_A1 + k5;
const auto B0 = TableLookupBytesOr0(v0, shuf_B0);
const auto B1 = TableLookupBytesOr0(v1, shuf_B1);
const auto B2 = TableLookupBytesOr0(v2, shuf_B2);
const VFromD<D> B = BitCast(d, B0 | B1 | B2);
const auto shuf_C0 = shuf_B2 + k6;
const auto shuf_C1 = shuf_B0 + k5;
const auto shuf_C2 = shuf_B1 + k5;
const auto C0 = TableLookupBytesOr0(v0, shuf_C0);
const auto C1 = TableLookupBytesOr0(v1, shuf_C1);
const auto C2 = TableLookupBytesOr0(v2, shuf_C2);
const VFromD<D> C = BitCast(d, C0 | C1 | C2);
detail::StoreTransposedBlocks3(A, B, C, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
const Repartition<uint8_t, decltype(d)> du8;
const auto k2 = Set(du8, uint8_t{2 * sizeof(TFromD<D>)});
const auto k3 = Set(du8, uint8_t{3 * sizeof(TFromD<D>)});
alignas(16) static constexpr uint8_t tbl_v1[16] = {
0x80, 0x80, 0, 1, 0x80, 0x80, 0x80, 0x80,
2, 3, 0x80, 0x80, 0x80, 0x80, 4, 5};
alignas(16) static constexpr uint8_t tbl_v2[16] = {
0x80, 0x80, 0x80, 0x80, 0, 1, 0x80, 0x80,
0x80, 0x80, 2, 3, 0x80, 0x80, 0x80, 0x80};
const auto shuf_A1 = LoadDup128(du8, tbl_v1);
const auto shuf_A0 = CombineShiftRightBytes<2>(du8, shuf_A1, shuf_A1);
const auto shuf_A2 = LoadDup128(du8, tbl_v2);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const VFromD<D> A = BitCast(d, A0 | A1 | A2);
const auto shuf_B0 = shuf_A1 + k3;
const auto shuf_B1 = shuf_A2 + k3;
const auto shuf_B2 = shuf_A0 + k2;
const auto B0 = TableLookupBytesOr0(v0, shuf_B0);
const auto B1 = TableLookupBytesOr0(v1, shuf_B1);
const auto B2 = TableLookupBytesOr0(v2, shuf_B2);
const VFromD<D> B = BitCast(d, B0 | B1 | B2);
const auto shuf_C0 = shuf_B1 + k3;
const auto shuf_C1 = shuf_B2 + k3;
const auto shuf_C2 = shuf_B0 + k2;
const auto C0 = TableLookupBytesOr0(v0, shuf_C0);
const auto C1 = TableLookupBytesOr0(v1, shuf_C1);
const auto C2 = TableLookupBytesOr0(v2, shuf_C2);
const VFromD<D> C = BitCast(d, C0 | C1 | C2);
detail::StoreTransposedBlocks3(A, B, C, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
const RepartitionToWide<decltype(d)> dw;
const VFromD<D> v10_v00 = InterleaveLower(d, v0, v1);
const VFromD<D> v01_v20 = OddEven(v0, v2);
const VFromD<D> A = BitCast(
d, InterleaveLower(dw, BitCast(dw, v10_v00), BitCast(dw, v01_v20)));
const VFromD<D> v1_321 = ShiftRightLanes<1>(d, v1);
const VFromD<D> v0_32 = ShiftRightLanes<2>(d, v0);
const VFromD<D> v21_v11 = OddEven(v2, v1_321);
const VFromD<D> v12_v02 = OddEven(v1_321, v0_32);
const VFromD<D> B = BitCast(
d, InterleaveLower(dw, BitCast(dw, v21_v11), BitCast(dw, v12_v02)));
const VFromD<D> v23_v13 = OddEven(v2, v1_321);
const VFromD<D> v03_v22 = OddEven(v0, v2);
const VFromD<D> C = BitCast(
d, InterleaveUpper(dw, BitCast(dw, v03_v22), BitCast(dw, v23_v13)));
detail::StoreTransposedBlocks3(A, B, C, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
const VFromD<D> A = InterleaveLower(d, v0, v1);
const VFromD<D> B = OddEven(v0, v2);
const VFromD<D> C = InterleaveUpper(d, v1, v2);
detail::StoreTransposedBlocks3(A, B, C, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, D d,
TFromD<D>* __restrict__ unaligned) {
constexpr size_t kFullN = 16 / sizeof(TFromD<D>);
const Full128<uint8_t> du;
const Full128<TFromD<D>> d_full;
const auto k5 = Set(du, uint8_t{5});
const auto k6 = Set(du, uint8_t{6});
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
alignas(16) static constexpr uint8_t tbl_v0[16] = {
0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80, 0x80,
3, 0x80, 0x80, 4, 0x80, 0x80, 5};
alignas(16) static constexpr uint8_t tbl_v1[16] = {
0x80, 0, 0x80, 0x80, 1, 0x80,
0x80, 2, 0x80, 0x80, 3, 0x80, 0x80, 4, 0x80, 0x80};
const auto shuf_A0 = Load(du, tbl_v0);
const auto shuf_A1 = Load(du, tbl_v1);
const auto shuf_A2 = CombineShiftRightBytes<15>(du, shuf_A1, shuf_A1);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const auto A = BitCast(d_full, A0 | A1 | A2);
StoreU(A, d_full, unaligned + 0 * kFullN);
const auto shuf_B0 = shuf_A2 + k6;
const auto shuf_B1 = shuf_A0 + k5;
const auto shuf_B2 = shuf_A1 + k5;
const auto B0 = TableLookupBytesOr0(v0, shuf_B0);
const auto B1 = TableLookupBytesOr0(v1, shuf_B1);
const auto B2 = TableLookupBytesOr0(v2, shuf_B2);
const VFromD<D> B{BitCast(d_full, B0 | B1 | B2).raw};
StoreU(B, d, unaligned + 1 * kFullN);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, D dh,
TFromD<D>* __restrict__ unaligned) {
const Twice<D> d_full;
const Full128<uint8_t> du8;
const auto k2 = Set(du8, uint8_t{2 * sizeof(TFromD<D>)});
const auto k3 = Set(du8, uint8_t{3 * sizeof(TFromD<D>)});
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
alignas(16) static constexpr uint8_t tbl_v1[16] = {
0x80, 0x80, 0, 1, 0x80, 0x80, 0x80, 0x80,
2, 3, 0x80, 0x80, 0x80, 0x80, 4, 5};
alignas(16) static constexpr uint8_t tbl_v2[16] = {
0x80, 0x80, 0x80, 0x80, 0, 1, 0x80, 0x80,
0x80, 0x80, 2, 3, 0x80, 0x80, 0x80, 0x80};
const auto shuf_A1 = Load(du8, tbl_v1);
const auto shuf_A0 = CombineShiftRightBytes<2>(du8, shuf_A1, shuf_A1);
const auto shuf_A2 = Load(du8, tbl_v2);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const VFromD<decltype(d_full)> A = BitCast(d_full, A0 | A1 | A2);
StoreU(A, d_full, unaligned);
const auto shuf_B0 = shuf_A1 + k3;
const auto shuf_B1 = shuf_A2 + k3;
const auto shuf_B2 = shuf_A0 + k2;
const auto B0 = TableLookupBytesOr0(v0, shuf_B0);
const auto B1 = TableLookupBytesOr0(v1, shuf_B1);
const auto B2 = TableLookupBytesOr0(v2, shuf_B2);
const VFromD<decltype(d_full)> B = BitCast(d_full, B0 | B1 | B2);
StoreU(VFromD<D>{B.raw}, dh, unaligned + MaxLanes(d_full));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
const VFromD<D> v10_v00 = InterleaveLower(d, v0, v1);
const VFromD<D> v01_v20 = OddEven(v0, v2);
const VFromD<D> v21_v11 = InterleaveUpper(d, v1, v2);
constexpr size_t kN = MaxLanes(d);
StoreU(v10_v00, d, unaligned + 0 * kN);
StoreU(v01_v20, d, unaligned + 1 * kN);
StoreU(v21_v11, d, unaligned + 2 * kN);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr,
hwy::EnableIf<(D::kPrivateLanes > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, D d,
TFromD<D>* __restrict__ unaligned) {
const Full128<uint8_t> du;
const Full128<TFromD<D>> d_full;
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
alignas(16) static constexpr uint8_t tbl_v0[16] = {
0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80,
0x80, 3, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};
const auto shuf_A0 = Load(du, tbl_v0);
const auto shuf_A1 = CombineShiftRightBytes<15>(du, shuf_A0, shuf_A0);
const auto shuf_A2 = CombineShiftRightBytes<14>(du, shuf_A0, shuf_A0);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const VFromD<decltype(d_full)> A = BitCast(d_full, A0 | A1 | A2);
alignas(16) TFromD<D> buf[MaxLanes(d_full)];
StoreU(A, d_full, buf);
CopyBytes<d.MaxBytes() * 3>(buf, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, D d,
TFromD<D>* __restrict__ unaligned) {
const Full128<uint8_t> du8;
const Full128<TFromD<D>> d_full;
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
alignas(16) static constexpr uint8_t tbl_v2[16] = {
0x80, 0x80, 0x80, 0x80, 0, 1, 0x80, 0x80,
0x80, 0x80, 2, 3, 0x80, 0x80, 0x80, 0x80};
const auto shuf_A2 =
Load(du8, tbl_v2);
const auto shuf_A1 =
CombineShiftRightBytes<2>(du8, shuf_A2, shuf_A2);
const auto shuf_A0 =
CombineShiftRightBytes<4>(du8, shuf_A2, shuf_A2);
const auto A0 = TableLookupBytesOr0(v0, shuf_A0);
const auto A1 = TableLookupBytesOr0(v1, shuf_A1);
const auto A2 = TableLookupBytesOr0(v2, shuf_A2);
const auto A = BitCast(d_full, A0 | A1 | A2);
alignas(16) TFromD<D> buf[MaxLanes(d_full)];
StoreU(A, d_full, buf);
CopyBytes<d.MaxBytes() * 3>(buf, unaligned);
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
StoreU(v0, d, unaligned + 0);
StoreU(v1, d, unaligned + 1);
StoreU(v2, d, unaligned + 2);
}
namespace detail {
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
inline __attribute__((always_inline)) void StoreTransposedBlocks4(VFromD<D> vA, VFromD<D> vB, VFromD<D> vC,
VFromD<D> vD, D d,
TFromD<D>* __restrict__ unaligned) {
constexpr size_t kN = MaxLanes(d);
StoreU(vA, d, unaligned + 0 * kN);
StoreU(vB, d, unaligned + 1 * kN);
StoreU(vC, d, unaligned + 2 * kN);
StoreU(vD, d, unaligned + 3 * kN);
}
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) != (8)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved4(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2,
VFromD<D> v3, D d,
TFromD<D>* __restrict__ unaligned) {
const RepartitionToWide<decltype(d)> dw;
const auto v10L = ZipLower(dw, v0, v1);
const auto v32L = ZipLower(dw, v2, v3);
const auto v10U = ZipUpper(dw, v0, v1);
const auto v32U = ZipUpper(dw, v2, v3);
const VFromD<D> vA = BitCast(d, InterleaveLower(dw, v10L, v32L));
const VFromD<D> vB = BitCast(d, InterleaveUpper(dw, v10L, v32L));
const VFromD<D> vC = BitCast(d, InterleaveLower(dw, v10U, v32U));
const VFromD<D> vD = BitCast(d, InterleaveUpper(dw, v10U, v32U));
detail::StoreTransposedBlocks4(vA, vB, vC, vD, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved4(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2,
VFromD<D> v3, D d,
TFromD<D>* __restrict__ unaligned) {
const VFromD<D> vA = InterleaveLower(d, v0, v1);
const VFromD<D> vB = InterleaveLower(d, v2, v3);
const VFromD<D> vC = InterleaveUpper(d, v0, v1);
const VFromD<D> vD = InterleaveUpper(d, v2, v3);
detail::StoreTransposedBlocks4(vA, vB, vC, vD, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) != (8)>* = nullptr, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) == 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved4(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, VFromD<D> part3, D ,
TFromD<D>* __restrict__ unaligned) {
const Full128<TFromD<D>> d_full;
const RepartitionToWide<decltype(d_full)> dw;
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
const VFromD<decltype(d_full)> v3{part3.raw};
const auto v10 = ZipLower(dw, v0, v1);
const auto v32 = ZipLower(dw, v2, v3);
const auto A = BitCast(d_full, InterleaveLower(dw, v10, v32));
const auto B = BitCast(d_full, InterleaveUpper(dw, v10, v32));
StoreU(A, d_full, unaligned);
StoreU(B, d_full, unaligned + MaxLanes(d_full));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved4(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, VFromD<D> part3, D ,
TFromD<D>* __restrict__ unaligned) {
const Full128<TFromD<D>> d_full;
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
const VFromD<decltype(d_full)> v3{part3.raw};
const auto A = InterleaveLower(d_full, v0, v1);
const auto B = InterleaveLower(d_full, v2, v3);
StoreU(A, d_full, unaligned);
StoreU(B, d_full, unaligned + MaxLanes(d_full));
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 4>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved4(VFromD<D> part0, VFromD<D> part1,
VFromD<D> part2, VFromD<D> part3, D d,
TFromD<D>* __restrict__ unaligned) {
const Full128<TFromD<D>> d_full;
const RepartitionToWide<decltype(d_full)> dw;
const VFromD<decltype(d_full)> v0{part0.raw};
const VFromD<decltype(d_full)> v1{part1.raw};
const VFromD<decltype(d_full)> v2{part2.raw};
const VFromD<decltype(d_full)> v3{part3.raw};
const auto v10 = ZipLower(dw, v0, v1);
const auto v32 = ZipLower(dw, v2, v3);
const auto v3210 = BitCast(d_full, InterleaveLower(dw, v10, v32));
alignas(16) TFromD<D> buf[MaxLanes(d_full)];
StoreU(v3210, d_full, buf);
CopyBytes<d.MaxBytes() * 4>(buf, unaligned);
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AbsDiff(V a, V b) {
return Sub(Max(a, b), Min(a, b));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedAdd(V a, V b) {
const DFromV<decltype(a)> d;
const auto sum = Add(a, b);
const auto overflow_mask =
MaskFromVec(BroadcastSignBit(AndNot(Xor(a, b), Xor(a, sum))));
const auto overflow_result =
Xor(BroadcastSignBit(a), Set(d, LimitsMax<int32_t>()));
return IfThenElse(overflow_mask, overflow_result, sum);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedSub(V a, V b) {
const DFromV<decltype(a)> d;
const auto diff = Sub(a, b);
const auto overflow_mask =
MaskFromVec(BroadcastSignBit(And(Xor(a, b), Xor(a, diff))));
const auto overflow_result =
Xor(BroadcastSignBit(a), Set(d, LimitsMax<int32_t>()));
return IfThenElse(overflow_mask, overflow_result, diff);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedAdd(V a, V b) {
const DFromV<decltype(a)> d;
const auto sum = Add(a, b);
const auto overflow_mask =
MaskFromVec(BroadcastSignBit(AndNot(Xor(a, b), Xor(a, sum))));
const auto overflow_result =
Xor(BroadcastSignBit(a), Set(d, LimitsMax<int64_t>()));
return IfThenElse(overflow_mask, overflow_result, sum);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedSub(V a, V b) {
const DFromV<decltype(a)> d;
const auto diff = Sub(a, b);
const auto overflow_mask =
MaskFromVec(BroadcastSignBit(And(Xor(a, b), Xor(a, diff))));
const auto overflow_result =
Xor(BroadcastSignBit(a), Set(d, LimitsMax<int64_t>()));
return IfThenElse(overflow_mask, overflow_result, diff);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedAdd(V a, V b) {
return Add(a, Min(b, Not(a)));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedSub(V a, V b) {
return Sub(a, Min(a, b));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedAdd(V a, V b) {
return Add(a, Min(b, Not(a)));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedSub(V a, V b) {
return Sub(a, Min(a, b));
}
template <class DN, hwy::EnableIf<IsSigned<TFromD<DN> >() && !IsFloat<TFromD<DN> >() && !IsSpecialFloat<TFromD<DN> >()>* = nullptr, class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
class V2 = VFromD<Rebind<TFromV<V>, DN>>,
hwy::EnableIf<(sizeof(TFromD<DN>) < sizeof(TFromV<V>))>* = nullptr,
hwy::EnableIf<(DFromV<V>::kPrivateLanes == DFromV<V2>::kPrivateLanes)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DN> DemoteTo(DN dn, V v) {
const DFromV<decltype(v)> d;
const RebindToSigned<decltype(d)> di;
const RebindToUnsigned<decltype(dn)> dn_u;
const auto i2i_demote_result = DemoteTo(dn, BitCast(di, v));
const auto max_signed_val = Set(dn, hwy::HighestValue<TFromD<DN>>());
return BitCast(
dn, Min(BitCast(dn_u, i2i_demote_result), BitCast(dn_u, max_signed_val)));
}
template <class DN, hwy::EnableIf<IsSigned<TFromD<DN> >() && !IsFloat<TFromD<DN> >() && !IsSpecialFloat<TFromD<DN> >()>* = nullptr, class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
class V2 = VFromD<Repartition<TFromV<V>, DN>>,
hwy::EnableIf<sizeof(TFromV<V>) == (sizeof(TFromD<DN>) * 2)>* = nullptr,
hwy::EnableIf<(DFromV<V>::kPrivateLanes == DFromV<V2>::kPrivateLanes)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DN> ReorderDemote2To(DN dn, V a, V b) {
const DFromV<decltype(a)> d;
const RebindToSigned<decltype(d)> di;
const RebindToUnsigned<decltype(dn)> dn_u;
const auto i2i_demote_result =
ReorderDemote2To(dn, BitCast(di, a), BitCast(di, b));
const auto max_signed_val = Set(dn, hwy::HighestValue<TFromD<DN>>());
return BitCast(
dn, Min(BitCast(dn_u, i2i_demote_result), BitCast(dn_u, max_signed_val)));
}
template <class DN, hwy::EnableIf<!IsSigned<TFromD<DN> >()>* = nullptr, class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
hwy::EnableIf<sizeof(TFromV<V>) == (sizeof(TFromD<DN>) * 2)>* = nullptr,
hwy::EnableIf<(DFromV<VFromD<DN>>::kPrivateLanes == DFromV<V>::kPrivateLanes * 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DN> OrderedTruncate2To(DN dn, V a, V b) {
return ConcatEven(dn, BitCast(dn, b), BitCast(dn, a));
}
namespace detail {
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const RebindToFloat<decltype(d)> df;
const RebindToSigned<decltype(d)> di;
const Repartition<int16_t, decltype(d)> di16;
const auto f32_bits = BitCast(d, ConvertTo(df, BitCast(di, v)));
return BitCast(d, Min(BitCast(di16, ShiftRight<23>(f32_bits)),
BitCast(di16, Set(d, 158))));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint32_t>()>* = nullptr>
inline __attribute__((always_inline)) V I32RangeU32ToF32BiasedExp(V v) {
const DFromV<decltype(v)> d;
const RebindToFloat<decltype(d)> df;
const RebindToSigned<decltype(d)> d_src;
const auto f32_bits = BitCast(d, ConvertTo(df, BitCast(d_src, v)));
return ShiftRight<23>(f32_bits);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes <= 16 / 4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Rebind<uint32_t, decltype(d)> du32;
const auto f32_biased_exp_as_u32 =
I32RangeU32ToF32BiasedExp(PromoteTo(du32, v));
return TruncateTo(d, f32_biased_exp_as_u32);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 16 / 4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Half<decltype(d)> dh;
const Rebind<uint32_t, decltype(dh)> du32;
const auto lo_u32 = PromoteTo(du32, LowerHalf(dh, v));
const auto hi_u32 = PromoteTo(du32, UpperHalf(dh, v));
const auto lo_f32_biased_exp_as_u32 = I32RangeU32ToF32BiasedExp(lo_u32);
const auto hi_f32_biased_exp_as_u32 = I32RangeU32ToF32BiasedExp(hi_u32);
const RebindToSigned<decltype(du32)> di32;
const RebindToSigned<decltype(d)> di;
return BitCast(d,
OrderedDemote2To(di, BitCast(di32, lo_f32_biased_exp_as_u32),
BitCast(di32, hi_f32_biased_exp_as_u32)));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes <= 16 / 4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Rebind<uint32_t, decltype(d)> du32;
const auto f32_biased_exp_as_u32 =
I32RangeU32ToF32BiasedExp(PromoteTo(du32, v));
return U8FromU32(f32_biased_exp_as_u32);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 16 / 4)>* = nullptr,
hwy::EnableIf<(D::kPrivateLanes <= 16 / 2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Half<decltype(d)> dh;
const Rebind<uint32_t, decltype(dh)> du32;
const Repartition<uint16_t, decltype(du32)> du16;
const auto lo_u32 = PromoteTo(du32, LowerHalf(dh, v));
const auto hi_u32 = PromoteTo(du32, UpperHalf(dh, v));
const auto lo_f32_biased_exp_as_u32 = I32RangeU32ToF32BiasedExp(lo_u32);
const auto hi_f32_biased_exp_as_u32 = I32RangeU32ToF32BiasedExp(hi_u32);
const RebindToSigned<decltype(du32)> di32;
const RebindToSigned<decltype(du16)> di16;
const auto f32_biased_exp_as_i16 =
OrderedDemote2To(di16, BitCast(di32, lo_f32_biased_exp_as_u32),
BitCast(di32, hi_f32_biased_exp_as_u32));
return DemoteTo(d, f32_biased_exp_as_i16);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 16 / 2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Half<decltype(d)> dh;
const Half<decltype(dh)> dq;
const Rebind<uint32_t, decltype(dq)> du32;
const Repartition<uint16_t, decltype(du32)> du16;
const auto lo_half = LowerHalf(dh, v);
const auto hi_half = UpperHalf(dh, v);
const auto u32_q0 = PromoteTo(du32, LowerHalf(dq, lo_half));
const auto u32_q1 = PromoteTo(du32, UpperHalf(dq, lo_half));
const auto u32_q2 = PromoteTo(du32, LowerHalf(dq, hi_half));
const auto u32_q3 = PromoteTo(du32, UpperHalf(dq, hi_half));
const auto f32_biased_exp_as_u32_q0 = I32RangeU32ToF32BiasedExp(u32_q0);
const auto f32_biased_exp_as_u32_q1 = I32RangeU32ToF32BiasedExp(u32_q1);
const auto f32_biased_exp_as_u32_q2 = I32RangeU32ToF32BiasedExp(u32_q2);
const auto f32_biased_exp_as_u32_q3 = I32RangeU32ToF32BiasedExp(u32_q3);
const RebindToSigned<decltype(du32)> di32;
const RebindToSigned<decltype(du16)> di16;
const auto lo_f32_biased_exp_as_i16 =
OrderedDemote2To(di16, BitCast(di32, f32_biased_exp_as_u32_q0),
BitCast(di32, f32_biased_exp_as_u32_q1));
const auto hi_f32_biased_exp_as_i16 =
OrderedDemote2To(di16, BitCast(di32, f32_biased_exp_as_u32_q2),
BitCast(di32, f32_biased_exp_as_u32_q3));
return OrderedDemote2To(d, lo_f32_biased_exp_as_i16,
hi_f32_biased_exp_as_i16);
}
template <class D>
using F32ExpLzcntMinMaxRepartition =
Repartition<UnsignedFromSize<((sizeof(TFromD<D>)) < (4) ? (sizeof(TFromD<D>)) : (4))>, D>;
template <class V>
using F32ExpLzcntMinMaxCmpV = VFromD<F32ExpLzcntMinMaxRepartition<DFromV<V>>>;
template <class V>
inline __attribute__((always_inline)) F32ExpLzcntMinMaxCmpV<V> F32ExpLzcntMinMaxBitCast(V v) {
const DFromV<decltype(v)> d;
const F32ExpLzcntMinMaxRepartition<decltype(d)> d2;
return BitCast(d2, v);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Repartition<uint32_t, decltype(d)> du32;
const auto f32_biased_exp = UIntToF32BiasedExp(du32, BitCast(du32, v));
const auto f32_biased_exp_adj =
IfThenZeroElse(Eq(f32_biased_exp, Zero(du32)),
BitCast(du32, Set(d, 0x0000002000000000u)));
const auto adj_f32_biased_exp = Add(f32_biased_exp, f32_biased_exp_adj);
return ShiftRight<32>(BitCast(
d, Max(F32ExpLzcntMinMaxBitCast(adj_f32_biased_exp),
F32ExpLzcntMinMaxBitCast(Reverse2(du32, adj_f32_biased_exp)))));
}
template <class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr>
inline __attribute__((always_inline)) V UIntToF32BiasedExp(V v) {
const DFromV<decltype(v)> d;
return UIntToF32BiasedExp(d, v);
}
template <class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2))) != 0>* = nullptr>
inline __attribute__((always_inline)) V NormalizeForUIntTruncConvToF32(V v) {
return v;
}
template <class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 4) | (1 << 8))) != 0>* = nullptr>
inline __attribute__((always_inline)) V NormalizeForUIntTruncConvToF32(V v) {
return AndNot(ShiftRight<24>(v), v);
}
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V HighestSetBitIndex(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
const auto f32_biased_exp = detail::UIntToF32BiasedExp(
detail::NormalizeForUIntTruncConvToF32(BitCast(du, v)));
return BitCast(d, Sub(f32_biased_exp, Set(du, TU{127})));
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V LeadingZeroCount(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
constexpr TU kNumOfBitsInT{sizeof(TU) * 8};
const auto f32_biased_exp = detail::UIntToF32BiasedExp(
detail::NormalizeForUIntTruncConvToF32(BitCast(du, v)));
const auto lz_count = Sub(Set(du, TU{kNumOfBitsInT + 126}), f32_biased_exp);
return BitCast(d,
Min(detail::F32ExpLzcntMinMaxBitCast(lz_count),
detail::F32ExpLzcntMinMaxBitCast(Set(du, kNumOfBitsInT))));
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V TrailingZeroCount(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const RebindToSigned<decltype(d)> di;
using TU = TFromD<decltype(du)>;
const auto vi = BitCast(di, v);
const auto lowest_bit = BitCast(du, And(vi, Neg(vi)));
constexpr TU kNumOfBitsInT{sizeof(TU) * 8};
const auto f32_biased_exp = detail::UIntToF32BiasedExp(lowest_bit);
const auto tz_count = Sub(f32_biased_exp, Set(du, TU{127}));
return BitCast(d,
Min(detail::F32ExpLzcntMinMaxBitCast(tz_count),
detail::F32ExpLzcntMinMaxBitCast(Set(du, kNumOfBitsInT))));
}
namespace detail {
template <class V>
inline __attribute__((always_inline)) V SubBytesMulInverseAndAffineLookup(V state, V affine_tblL,
V affine_tblU) {
const DFromV<V> du;
const auto mask = Set(du, uint8_t{0xF});
{
alignas(16) static constexpr uint8_t basisL[16] = {
0x00, 0x70, 0x2A, 0x5A, 0x98, 0xE8, 0xB2, 0xC2,
0x08, 0x78, 0x22, 0x52, 0x90, 0xE0, 0xBA, 0xCA};
alignas(16) static constexpr uint8_t basisU[16] = {
0x00, 0x4D, 0x7C, 0x31, 0x7D, 0x30, 0x01, 0x4C,
0x81, 0xCC, 0xFD, 0xB0, 0xFC, 0xB1, 0x80, 0xCD};
const auto sL = And(state, mask);
const auto sU = ShiftRight<4>(state);
const auto gf4L = TableLookupBytes(LoadDup128(du, basisL), sL);
const auto gf4U = TableLookupBytes(LoadDup128(du, basisU), sU);
state = Xor(gf4L, gf4U);
}
alignas(16) static constexpr uint8_t kZetaInv[16] = {
0x80, 7, 11, 15, 6, 10, 4, 1, 9, 8, 5, 2, 12, 14, 13, 3};
alignas(16) static constexpr uint8_t kInv[16] = {
0x80, 1, 8, 13, 15, 6, 5, 14, 2, 12, 11, 10, 9, 3, 7, 4};
const auto tbl = LoadDup128(du, kInv);
const auto sL = And(state, mask);
const auto sU = ShiftRight<4>(state);
const auto sX = Xor(sU, sL);
const auto invL = TableLookupBytes(LoadDup128(du, kZetaInv), sL);
const auto invU = TableLookupBytes(tbl, sU);
const auto invX = TableLookupBytes(tbl, sX);
const auto outL = Xor(sX, TableLookupBytesOr0(tbl, Xor(invL, invU)));
const auto outU = Xor(sU, TableLookupBytesOr0(tbl, Xor(invL, invX)));
const auto affL = TableLookupBytesOr0(affine_tblL, outL);
const auto affU = TableLookupBytesOr0(affine_tblU, outU);
return Xor(affL, affU);
}
template <class V>
inline __attribute__((always_inline)) V SubBytes(V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t kAffineL[16] = {
0x00, 0xC7, 0xBD, 0x6F, 0x17, 0x6D, 0xD2, 0xD0,
0x78, 0xA8, 0x02, 0xC5, 0x7A, 0xBF, 0xAA, 0x15};
alignas(16) static constexpr uint8_t kAffineU[16] = {
0x00, 0x6A, 0xBB, 0x5F, 0xA5, 0x74, 0xE4, 0xCF,
0xFA, 0x35, 0x2B, 0x41, 0xD1, 0x90, 0x1E, 0x8E};
return Xor(SubBytesMulInverseAndAffineLookup(state, LoadDup128(du, kAffineL),
LoadDup128(du, kAffineU)),
Set(du, uint8_t{0x63}));
}
template <class V>
inline __attribute__((always_inline)) V InvSubBytes(V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t kGF2P4InvToGF2P8InvL[16]{
0x00, 0x40, 0xF9, 0x7E, 0x53, 0xEA, 0x87, 0x13,
0x2D, 0x3E, 0x94, 0xD4, 0xB9, 0x6D, 0xAA, 0xC7};
alignas(16) static constexpr uint8_t kGF2P4InvToGF2P8InvU[16]{
0x00, 0x1D, 0x44, 0x93, 0x0F, 0x56, 0xD7, 0x12,
0x9C, 0x8E, 0xC5, 0xD8, 0x59, 0x81, 0x4B, 0xCA};
const auto b = Xor(Xor3(Or(ShiftLeft<1>(state), ShiftRight<7>(state)),
Or(ShiftLeft<3>(state), ShiftRight<5>(state)),
Or(ShiftLeft<6>(state), ShiftRight<2>(state))),
Set(du, uint8_t{0x05}));
return SubBytesMulInverseAndAffineLookup(
b, LoadDup128(du, kGF2P4InvToGF2P8InvL),
LoadDup128(du, kGF2P4InvToGF2P8InvU));
}
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr,
hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr, void* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
alignas(16) constexpr uint8_t kLookup[16] = {
0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
};
const auto lo = And(v, Set(d, uint8_t{0xF}));
const auto hi = ShiftRight<4>(v);
const auto lookup = LoadDup128(d, kLookup);
return Add(TableLookupBytes(lookup, hi), TableLookupBytes(lookup, lo));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
const V k33 = Set(d, uint8_t{0x33});
v = Sub(v, And(ShiftRight<1>(v), Set(d, uint8_t{0x55})));
v = Add(And(ShiftRight<2>(v), k33), And(v, k33));
return And(Add(v, ShiftRight<4>(v)), Set(d, uint8_t{0x0F}));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
const Repartition<uint8_t, decltype(d)> d8;
const auto vals = BitCast(d, PopulationCount(BitCast(d8, v)));
return Add(ShiftRight<8>(vals), And(vals, Set(d, uint16_t{0xFF})));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
Repartition<uint16_t, decltype(d)> d16;
auto vals = BitCast(d, PopulationCount(BitCast(d16, v)));
return Add(ShiftRight<16>(vals), And(vals, Set(d, uint32_t{0xFF})));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
Repartition<uint32_t, decltype(d)> d32;
auto vals = BitCast(d, PopulationCount(BitCast(d32, v)));
return Add(ShiftRight<32>(vals), And(vals, Set(d, 0xFFULL)));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (1)>* = nullptr,
hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16 / 2>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V operator*(const V a, const V b) {
const DFromV<decltype(a)> d;
const Rebind<MakeWide<TFromV<V>>, decltype(d)> dw;
const RebindToUnsigned<decltype(d)> du;
const RebindToUnsigned<decltype(dw)> dwu;
const VFromD<decltype(dw)> mul = PromoteTo(dw, a) * PromoteTo(dw, b);
return BitCast(d, TruncateTo(du, BitCast(dwu, mul)));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (1)>* = nullptr,
hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) > 16 / 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V operator*(const V a, const V b) {
const DFromV<decltype(a)> d;
const Half<decltype(d)> dh;
const Twice<RepartitionToWide<decltype(dh)>> dw;
const VFromD<decltype(dw)> a0 = PromoteTo(dw, LowerHalf(dh, a));
const VFromD<decltype(dw)> a1 = PromoteTo(dw, UpperHalf(dh, a));
const VFromD<decltype(dw)> b0 = PromoteTo(dw, LowerHalf(dh, b));
const VFromD<decltype(dw)> b1 = PromoteTo(dw, UpperHalf(dh, b));
const VFromD<decltype(dw)> m0 = a0 * b0;
const VFromD<decltype(dw)> m1 = a1 * b1;
return ConcatEven(d, BitCast(d, m1), BitCast(d, m0));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (8)>* = nullptr, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) == 8>* = nullptr,
hwy::EnableIf<!hwy::IsFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V operator*(V x, V y) {
const DFromV<V> d;
using T = TFromD<decltype(d)>;
using TU = MakeUnsigned<T>;
const TU xu = static_cast<TU>(GetLane(x));
const TU yu = static_cast<TU>(GetLane(y));
return Set(d, static_cast<T>(xu * yu));
}
template <class V, class D64 = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D64>, uint64_t>()>* = nullptr,
hwy::EnableIf<(D64::kPrivateLanes * sizeof(TFromD<D64>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V operator*(V x, V y) {
RepartitionToNarrow<D64> d32;
auto x32 = BitCast(d32, x);
auto y32 = BitCast(d32, y);
auto lolo = BitCast(d32, MulEven(x32, y32));
auto lohi = BitCast(d32, MulEven(x32, BitCast(d32, ShiftRight<32>(y))));
auto hilo = BitCast(d32, MulEven(BitCast(d32, ShiftRight<32>(x)), y32));
auto hi = BitCast(d32, ShiftLeft<32>(BitCast(D64{}, lohi + hilo)));
return BitCast(D64{}, lolo + hi);
}
template <class V, class DI64 = DFromV<V>, hwy::EnableIf<IsSame<TFromD<DI64>, int64_t>()>* = nullptr,
hwy::EnableIf<(DI64::kPrivateLanes * sizeof(TFromD<DI64>) > 8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V operator*(V x, V y) {
RebindToUnsigned<DI64> du64;
return BitCast(DI64{}, BitCast(du64, x) * BitCast(du64, y));
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V MulAdd(V mul, V x, V add) {
return Add(Mul(mul, x), add);
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V NegMulAdd(V mul, V x, V add) {
return Sub(add, Mul(mul, x));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ApproximateReciprocal(V v) {
const DFromV<decltype(v)> d;
return Div(Set(d, 1.0), v);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ApproximateReciprocalSqrt(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const auto half = Mul(v, Set(d, 0.5));
const auto guess = BitCast(d, Sub(Set(du, uint64_t{0x5FE6EB50C7B537A9u}),
ShiftRight<1>(BitCast(du, v))));
return Mul(guess, NegMulAdd(Mul(half, guess), guess, Set(d, 1.5)));
}
template <class V, class D, typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressBitsStore(V v, const uint8_t* __restrict__ bits, D d,
T* unaligned) {
alignas(16) T lanes[MaxLanes(d)];
Store(v, d, lanes);
const Simd<T, ((MaxLanes(d)) < (8) ? (MaxLanes(d)) : (8)), 0> d8;
T* __restrict__ pos = unaligned;
alignas(16) constexpr T table[2048] = {
0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
1, 0, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
2, 0, 1, 3, 4, 5, 6, 7, 0, 2, 1, 3, 4, 5, 6, 7,
1, 2, 0, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
3, 0, 1, 2, 4, 5, 6, 7, 0, 3, 1, 2, 4, 5, 6, 7,
1, 3, 0, 2, 4, 5, 6, 7, 0, 1, 3, 2, 4, 5, 6, 7,
2, 3, 0, 1, 4, 5, 6, 7, 0, 2, 3, 1, 4, 5, 6, 7,
1, 2, 3, 0, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
4, 0, 1, 2, 3, 5, 6, 7, 0, 4, 1, 2, 3, 5, 6, 7,
1, 4, 0, 2, 3, 5, 6, 7, 0, 1, 4, 2, 3, 5, 6, 7,
2, 4, 0, 1, 3, 5, 6, 7, 0, 2, 4, 1, 3, 5, 6, 7,
1, 2, 4, 0, 3, 5, 6, 7, 0, 1, 2, 4, 3, 5, 6, 7,
3, 4, 0, 1, 2, 5, 6, 7, 0, 3, 4, 1, 2, 5, 6, 7,
1, 3, 4, 0, 2, 5, 6, 7, 0, 1, 3, 4, 2, 5, 6, 7,
2, 3, 4, 0, 1, 5, 6, 7, 0, 2, 3, 4, 1, 5, 6, 7,
1, 2, 3, 4, 0, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
5, 0, 1, 2, 3, 4, 6, 7, 0, 5, 1, 2, 3, 4, 6, 7,
1, 5, 0, 2, 3, 4, 6, 7, 0, 1, 5, 2, 3, 4, 6, 7,
2, 5, 0, 1, 3, 4, 6, 7, 0, 2, 5, 1, 3, 4, 6, 7,
1, 2, 5, 0, 3, 4, 6, 7, 0, 1, 2, 5, 3, 4, 6, 7,
3, 5, 0, 1, 2, 4, 6, 7, 0, 3, 5, 1, 2, 4, 6, 7,
1, 3, 5, 0, 2, 4, 6, 7, 0, 1, 3, 5, 2, 4, 6, 7,
2, 3, 5, 0, 1, 4, 6, 7, 0, 2, 3, 5, 1, 4, 6, 7,
1, 2, 3, 5, 0, 4, 6, 7, 0, 1, 2, 3, 5, 4, 6, 7,
4, 5, 0, 1, 2, 3, 6, 7, 0, 4, 5, 1, 2, 3, 6, 7,
1, 4, 5, 0, 2, 3, 6, 7, 0, 1, 4, 5, 2, 3, 6, 7,
2, 4, 5, 0, 1, 3, 6, 7, 0, 2, 4, 5, 1, 3, 6, 7,
1, 2, 4, 5, 0, 3, 6, 7, 0, 1, 2, 4, 5, 3, 6, 7,
3, 4, 5, 0, 1, 2, 6, 7, 0, 3, 4, 5, 1, 2, 6, 7,
1, 3, 4, 5, 0, 2, 6, 7, 0, 1, 3, 4, 5, 2, 6, 7,
2, 3, 4, 5, 0, 1, 6, 7, 0, 2, 3, 4, 5, 1, 6, 7,
1, 2, 3, 4, 5, 0, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
6, 0, 1, 2, 3, 4, 5, 7, 0, 6, 1, 2, 3, 4, 5, 7,
1, 6, 0, 2, 3, 4, 5, 7, 0, 1, 6, 2, 3, 4, 5, 7,
2, 6, 0, 1, 3, 4, 5, 7, 0, 2, 6, 1, 3, 4, 5, 7,
1, 2, 6, 0, 3, 4, 5, 7, 0, 1, 2, 6, 3, 4, 5, 7,
3, 6, 0, 1, 2, 4, 5, 7, 0, 3, 6, 1, 2, 4, 5, 7,
1, 3, 6, 0, 2, 4, 5, 7, 0, 1, 3, 6, 2, 4, 5, 7,
2, 3, 6, 0, 1, 4, 5, 7, 0, 2, 3, 6, 1, 4, 5, 7,
1, 2, 3, 6, 0, 4, 5, 7, 0, 1, 2, 3, 6, 4, 5, 7,
4, 6, 0, 1, 2, 3, 5, 7, 0, 4, 6, 1, 2, 3, 5, 7,
1, 4, 6, 0, 2, 3, 5, 7, 0, 1, 4, 6, 2, 3, 5, 7,
2, 4, 6, 0, 1, 3, 5, 7, 0, 2, 4, 6, 1, 3, 5, 7,
1, 2, 4, 6, 0, 3, 5, 7, 0, 1, 2, 4, 6, 3, 5, 7,
3, 4, 6, 0, 1, 2, 5, 7, 0, 3, 4, 6, 1, 2, 5, 7,
1, 3, 4, 6, 0, 2, 5, 7, 0, 1, 3, 4, 6, 2, 5, 7,
2, 3, 4, 6, 0, 1, 5, 7, 0, 2, 3, 4, 6, 1, 5, 7,
1, 2, 3, 4, 6, 0, 5, 7, 0, 1, 2, 3, 4, 6, 5, 7,
5, 6, 0, 1, 2, 3, 4, 7, 0, 5, 6, 1, 2, 3, 4, 7,
1, 5, 6, 0, 2, 3, 4, 7, 0, 1, 5, 6, 2, 3, 4, 7,
2, 5, 6, 0, 1, 3, 4, 7, 0, 2, 5, 6, 1, 3, 4, 7,
1, 2, 5, 6, 0, 3, 4, 7, 0, 1, 2, 5, 6, 3, 4, 7,
3, 5, 6, 0, 1, 2, 4, 7, 0, 3, 5, 6, 1, 2, 4, 7,
1, 3, 5, 6, 0, 2, 4, 7, 0, 1, 3, 5, 6, 2, 4, 7,
2, 3, 5, 6, 0, 1, 4, 7, 0, 2, 3, 5, 6, 1, 4, 7,
1, 2, 3, 5, 6, 0, 4, 7, 0, 1, 2, 3, 5, 6, 4, 7,
4, 5, 6, 0, 1, 2, 3, 7, 0, 4, 5, 6, 1, 2, 3, 7,
1, 4, 5, 6, 0, 2, 3, 7, 0, 1, 4, 5, 6, 2, 3, 7,
2, 4, 5, 6, 0, 1, 3, 7, 0, 2, 4, 5, 6, 1, 3, 7,
1, 2, 4, 5, 6, 0, 3, 7, 0, 1, 2, 4, 5, 6, 3, 7,
3, 4, 5, 6, 0, 1, 2, 7, 0, 3, 4, 5, 6, 1, 2, 7,
1, 3, 4, 5, 6, 0, 2, 7, 0, 1, 3, 4, 5, 6, 2, 7,
2, 3, 4, 5, 6, 0, 1, 7, 0, 2, 3, 4, 5, 6, 1, 7,
1, 2, 3, 4, 5, 6, 0, 7, 0, 1, 2, 3, 4, 5, 6, 7,
7, 0, 1, 2, 3, 4, 5, 6, 0, 7, 1, 2, 3, 4, 5, 6,
1, 7, 0, 2, 3, 4, 5, 6, 0, 1, 7, 2, 3, 4, 5, 6,
2, 7, 0, 1, 3, 4, 5, 6, 0, 2, 7, 1, 3, 4, 5, 6,
1, 2, 7, 0, 3, 4, 5, 6, 0, 1, 2, 7, 3, 4, 5, 6,
3, 7, 0, 1, 2, 4, 5, 6, 0, 3, 7, 1, 2, 4, 5, 6,
1, 3, 7, 0, 2, 4, 5, 6, 0, 1, 3, 7, 2, 4, 5, 6,
2, 3, 7, 0, 1, 4, 5, 6, 0, 2, 3, 7, 1, 4, 5, 6,
1, 2, 3, 7, 0, 4, 5, 6, 0, 1, 2, 3, 7, 4, 5, 6,
4, 7, 0, 1, 2, 3, 5, 6, 0, 4, 7, 1, 2, 3, 5, 6,
1, 4, 7, 0, 2, 3, 5, 6, 0, 1, 4, 7, 2, 3, 5, 6,
2, 4, 7, 0, 1, 3, 5, 6, 0, 2, 4, 7, 1, 3, 5, 6,
1, 2, 4, 7, 0, 3, 5, 6, 0, 1, 2, 4, 7, 3, 5, 6,
3, 4, 7, 0, 1, 2, 5, 6, 0, 3, 4, 7, 1, 2, 5, 6,
1, 3, 4, 7, 0, 2, 5, 6, 0, 1, 3, 4, 7, 2, 5, 6,
2, 3, 4, 7, 0, 1, 5, 6, 0, 2, 3, 4, 7, 1, 5, 6,
1, 2, 3, 4, 7, 0, 5, 6, 0, 1, 2, 3, 4, 7, 5, 6,
5, 7, 0, 1, 2, 3, 4, 6, 0, 5, 7, 1, 2, 3, 4, 6,
1, 5, 7, 0, 2, 3, 4, 6, 0, 1, 5, 7, 2, 3, 4, 6,
2, 5, 7, 0, 1, 3, 4, 6, 0, 2, 5, 7, 1, 3, 4, 6,
1, 2, 5, 7, 0, 3, 4, 6, 0, 1, 2, 5, 7, 3, 4, 6,
3, 5, 7, 0, 1, 2, 4, 6, 0, 3, 5, 7, 1, 2, 4, 6,
1, 3, 5, 7, 0, 2, 4, 6, 0, 1, 3, 5, 7, 2, 4, 6,
2, 3, 5, 7, 0, 1, 4, 6, 0, 2, 3, 5, 7, 1, 4, 6,
1, 2, 3, 5, 7, 0, 4, 6, 0, 1, 2, 3, 5, 7, 4, 6,
4, 5, 7, 0, 1, 2, 3, 6, 0, 4, 5, 7, 1, 2, 3, 6,
1, 4, 5, 7, 0, 2, 3, 6, 0, 1, 4, 5, 7, 2, 3, 6,
2, 4, 5, 7, 0, 1, 3, 6, 0, 2, 4, 5, 7, 1, 3, 6,
1, 2, 4, 5, 7, 0, 3, 6, 0, 1, 2, 4, 5, 7, 3, 6,
3, 4, 5, 7, 0, 1, 2, 6, 0, 3, 4, 5, 7, 1, 2, 6,
1, 3, 4, 5, 7, 0, 2, 6, 0, 1, 3, 4, 5, 7, 2, 6,
2, 3, 4, 5, 7, 0, 1, 6, 0, 2, 3, 4, 5, 7, 1, 6,
1, 2, 3, 4, 5, 7, 0, 6, 0, 1, 2, 3, 4, 5, 7, 6,
6, 7, 0, 1, 2, 3, 4, 5, 0, 6, 7, 1, 2, 3, 4, 5,
1, 6, 7, 0, 2, 3, 4, 5, 0, 1, 6, 7, 2, 3, 4, 5,
2, 6, 7, 0, 1, 3, 4, 5, 0, 2, 6, 7, 1, 3, 4, 5,
1, 2, 6, 7, 0, 3, 4, 5, 0, 1, 2, 6, 7, 3, 4, 5,
3, 6, 7, 0, 1, 2, 4, 5, 0, 3, 6, 7, 1, 2, 4, 5,
1, 3, 6, 7, 0, 2, 4, 5, 0, 1, 3, 6, 7, 2, 4, 5,
2, 3, 6, 7, 0, 1, 4, 5, 0, 2, 3, 6, 7, 1, 4, 5,
1, 2, 3, 6, 7, 0, 4, 5, 0, 1, 2, 3, 6, 7, 4, 5,
4, 6, 7, 0, 1, 2, 3, 5, 0, 4, 6, 7, 1, 2, 3, 5,
1, 4, 6, 7, 0, 2, 3, 5, 0, 1, 4, 6, 7, 2, 3, 5,
2, 4, 6, 7, 0, 1, 3, 5, 0, 2, 4, 6, 7, 1, 3, 5,
1, 2, 4, 6, 7, 0, 3, 5, 0, 1, 2, 4, 6, 7, 3, 5,
3, 4, 6, 7, 0, 1, 2, 5, 0, 3, 4, 6, 7, 1, 2, 5,
1, 3, 4, 6, 7, 0, 2, 5, 0, 1, 3, 4, 6, 7, 2, 5,
2, 3, 4, 6, 7, 0, 1, 5, 0, 2, 3, 4, 6, 7, 1, 5,
1, 2, 3, 4, 6, 7, 0, 5, 0, 1, 2, 3, 4, 6, 7, 5,
5, 6, 7, 0, 1, 2, 3, 4, 0, 5, 6, 7, 1, 2, 3, 4,
1, 5, 6, 7, 0, 2, 3, 4, 0, 1, 5, 6, 7, 2, 3, 4,
2, 5, 6, 7, 0, 1, 3, 4, 0, 2, 5, 6, 7, 1, 3, 4,
1, 2, 5, 6, 7, 0, 3, 4, 0, 1, 2, 5, 6, 7, 3, 4,
3, 5, 6, 7, 0, 1, 2, 4, 0, 3, 5, 6, 7, 1, 2, 4,
1, 3, 5, 6, 7, 0, 2, 4, 0, 1, 3, 5, 6, 7, 2, 4,
2, 3, 5, 6, 7, 0, 1, 4, 0, 2, 3, 5, 6, 7, 1, 4,
1, 2, 3, 5, 6, 7, 0, 4, 0, 1, 2, 3, 5, 6, 7, 4,
4, 5, 6, 7, 0, 1, 2, 3, 0, 4, 5, 6, 7, 1, 2, 3,
1, 4, 5, 6, 7, 0, 2, 3, 0, 1, 4, 5, 6, 7, 2, 3,
2, 4, 5, 6, 7, 0, 1, 3, 0, 2, 4, 5, 6, 7, 1, 3,
1, 2, 4, 5, 6, 7, 0, 3, 0, 1, 2, 4, 5, 6, 7, 3,
3, 4, 5, 6, 7, 0, 1, 2, 0, 3, 4, 5, 6, 7, 1, 2,
1, 3, 4, 5, 6, 7, 0, 2, 0, 1, 3, 4, 5, 6, 7, 2,
2, 3, 4, 5, 6, 7, 0, 1, 0, 2, 3, 4, 5, 6, 7, 1,
1, 2, 3, 4, 5, 6, 7, 0, 0, 1, 2, 3, 4, 5, 6, 7};
for (size_t i = 0; i < Lanes(d); i += 8) {
const size_t bits8 = bits[i / 8];
const auto indices = Load(d8, table + bits8 * 8);
const auto compressed = TableLookupBytes(LoadU(d8, lanes + i), indices);
StoreU(compressed, d8, pos);
pos += PopCount(bits8);
}
return static_cast<size_t>(pos - unaligned);
}
template <class V, class M, class D, typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressStore(V v, M mask, D d, T* __restrict__ unaligned) {
uint8_t bits[((size_t{8}) > (MaxLanes(d) / 8) ? (size_t{8}) : (MaxLanes(d) / 8))];
(void)StoreMaskBits(d, mask, bits);
return CompressBitsStore(v, bits, d, unaligned);
}
template <class V, class M, class D, typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressBlendedStore(V v, M mask, D d,
T* __restrict__ unaligned) {
alignas(16) T buf[MaxLanes(d)];
const size_t bytes = CompressStore(v, mask, d, buf);
BlendedStore(Load(d, buf), FirstN(d, bytes), d, unaligned);
return bytes;
}
template <class V, class M, typename T = TFromV<V>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Compress(V v, const M mask) {
const DFromV<V> d;
alignas(16) T lanes[MaxLanes(d)];
(void)CompressStore(v, mask, d, lanes);
return Load(d, lanes);
}
template <class V, typename T = TFromV<V>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CompressBits(V v, const uint8_t* __restrict__ bits) {
const DFromV<V> d;
alignas(16) T lanes[MaxLanes(d)];
(void)CompressBitsStore(v, bits, d, lanes);
return Load(d, lanes);
}
template <class V, class M, typename T = TFromV<V>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CompressNot(V v, M mask) {
return Compress(v, Not(mask));
}
namespace detail {
template <size_t N>
inline __attribute__((always_inline)) Vec128<uint8_t, N> IndicesForExpandFromBits(uint64_t mask_bits) {
static_assert(N <= 8, "Should only be called for half-vectors");
const Simd<uint8_t, N, 0> du8;
do { } while (0);
alignas(16) static constexpr uint8_t table[2048] = {
128, 128, 128, 128, 128, 128, 128, 128,
0, 128, 128, 128, 128, 128, 128, 128,
128, 0, 128, 128, 128, 128, 128, 128,
0, 1, 128, 128, 128, 128, 128, 128,
128, 128, 0, 128, 128, 128, 128, 128,
0, 128, 1, 128, 128, 128, 128, 128,
128, 0, 1, 128, 128, 128, 128, 128,
0, 1, 2, 128, 128, 128, 128, 128,
128, 128, 128, 0, 128, 128, 128, 128,
0, 128, 128, 1, 128, 128, 128, 128,
128, 0, 128, 1, 128, 128, 128, 128,
0, 1, 128, 2, 128, 128, 128, 128,
128, 128, 0, 1, 128, 128, 128, 128,
0, 128, 1, 2, 128, 128, 128, 128,
128, 0, 1, 2, 128, 128, 128, 128,
0, 1, 2, 3, 128, 128, 128, 128,
128, 128, 128, 128, 0, 128, 128, 128,
0, 128, 128, 128, 1, 128, 128, 128,
128, 0, 128, 128, 1, 128, 128, 128,
0, 1, 128, 128, 2, 128, 128, 128,
128, 128, 0, 128, 1, 128, 128, 128,
0, 128, 1, 128, 2, 128, 128, 128,
128, 0, 1, 128, 2, 128, 128, 128,
0, 1, 2, 128, 3, 128, 128, 128,
128, 128, 128, 0, 1, 128, 128, 128,
0, 128, 128, 1, 2, 128, 128, 128,
128, 0, 128, 1, 2, 128, 128, 128,
0, 1, 128, 2, 3, 128, 128, 128,
128, 128, 0, 1, 2, 128, 128, 128,
0, 128, 1, 2, 3, 128, 128, 128,
128, 0, 1, 2, 3, 128, 128, 128,
0, 1, 2, 3, 4, 128, 128, 128,
128, 128, 128, 128, 128, 0, 128, 128,
0, 128, 128, 128, 128, 1, 128, 128,
128, 0, 128, 128, 128, 1, 128, 128,
0, 1, 128, 128, 128, 2, 128, 128,
128, 128, 0, 128, 128, 1, 128, 128,
0, 128, 1, 128, 128, 2, 128, 128,
128, 0, 1, 128, 128, 2, 128, 128,
0, 1, 2, 128, 128, 3, 128, 128,
128, 128, 128, 0, 128, 1, 128, 128,
0, 128, 128, 1, 128, 2, 128, 128,
128, 0, 128, 1, 128, 2, 128, 128,
0, 1, 128, 2, 128, 3, 128, 128,
128, 128, 0, 1, 128, 2, 128, 128,
0, 128, 1, 2, 128, 3, 128, 128,
128, 0, 1, 2, 128, 3, 128, 128,
0, 1, 2, 3, 128, 4, 128, 128,
128, 128, 128, 128, 0, 1, 128, 128,
0, 128, 128, 128, 1, 2, 128, 128,
128, 0, 128, 128, 1, 2, 128, 128,
0, 1, 128, 128, 2, 3, 128, 128,
128, 128, 0, 128, 1, 2, 128, 128,
0, 128, 1, 128, 2, 3, 128, 128,
128, 0, 1, 128, 2, 3, 128, 128,
0, 1, 2, 128, 3, 4, 128, 128,
128, 128, 128, 0, 1, 2, 128, 128,
0, 128, 128, 1, 2, 3, 128, 128,
128, 0, 128, 1, 2, 3, 128, 128,
0, 1, 128, 2, 3, 4, 128, 128,
128, 128, 0, 1, 2, 3, 128, 128,
0, 128, 1, 2, 3, 4, 128, 128,
128, 0, 1, 2, 3, 4, 128, 128,
0, 1, 2, 3, 4, 5, 128, 128,
128, 128, 128, 128, 128, 128, 0, 128,
0, 128, 128, 128, 128, 128, 1, 128,
128, 0, 128, 128, 128, 128, 1, 128,
0, 1, 128, 128, 128, 128, 2, 128,
128, 128, 0, 128, 128, 128, 1, 128,
0, 128, 1, 128, 128, 128, 2, 128,
128, 0, 1, 128, 128, 128, 2, 128,
0, 1, 2, 128, 128, 128, 3, 128,
128, 128, 128, 0, 128, 128, 1, 128,
0, 128, 128, 1, 128, 128, 2, 128,
128, 0, 128, 1, 128, 128, 2, 128,
0, 1, 128, 2, 128, 128, 3, 128,
128, 128, 0, 1, 128, 128, 2, 128,
0, 128, 1, 2, 128, 128, 3, 128,
128, 0, 1, 2, 128, 128, 3, 128,
0, 1, 2, 3, 128, 128, 4, 128,
128, 128, 128, 128, 0, 128, 1, 128,
0, 128, 128, 128, 1, 128, 2, 128,
128, 0, 128, 128, 1, 128, 2, 128,
0, 1, 128, 128, 2, 128, 3, 128,
128, 128, 0, 128, 1, 128, 2, 128,
0, 128, 1, 128, 2, 128, 3, 128,
128, 0, 1, 128, 2, 128, 3, 128,
0, 1, 2, 128, 3, 128, 4, 128,
128, 128, 128, 0, 1, 128, 2, 128,
0, 128, 128, 1, 2, 128, 3, 128,
128, 0, 128, 1, 2, 128, 3, 128,
0, 1, 128, 2, 3, 128, 4, 128,
128, 128, 0, 1, 2, 128, 3, 128,
0, 128, 1, 2, 3, 128, 4, 128,
128, 0, 1, 2, 3, 128, 4, 128,
0, 1, 2, 3, 4, 128, 5, 128,
128, 128, 128, 128, 128, 0, 1, 128,
0, 128, 128, 128, 128, 1, 2, 128,
128, 0, 128, 128, 128, 1, 2, 128,
0, 1, 128, 128, 128, 2, 3, 128,
128, 128, 0, 128, 128, 1, 2, 128,
0, 128, 1, 128, 128, 2, 3, 128,
128, 0, 1, 128, 128, 2, 3, 128,
0, 1, 2, 128, 128, 3, 4, 128,
128, 128, 128, 0, 128, 1, 2, 128,
0, 128, 128, 1, 128, 2, 3, 128,
128, 0, 128, 1, 128, 2, 3, 128,
0, 1, 128, 2, 128, 3, 4, 128,
128, 128, 0, 1, 128, 2, 3, 128,
0, 128, 1, 2, 128, 3, 4, 128,
128, 0, 1, 2, 128, 3, 4, 128,
0, 1, 2, 3, 128, 4, 5, 128,
128, 128, 128, 128, 0, 1, 2, 128,
0, 128, 128, 128, 1, 2, 3, 128,
128, 0, 128, 128, 1, 2, 3, 128,
0, 1, 128, 128, 2, 3, 4, 128,
128, 128, 0, 128, 1, 2, 3, 128,
0, 128, 1, 128, 2, 3, 4, 128,
128, 0, 1, 128, 2, 3, 4, 128,
0, 1, 2, 128, 3, 4, 5, 128,
128, 128, 128, 0, 1, 2, 3, 128,
0, 128, 128, 1, 2, 3, 4, 128,
128, 0, 128, 1, 2, 3, 4, 128,
0, 1, 128, 2, 3, 4, 5, 128,
128, 128, 0, 1, 2, 3, 4, 128,
0, 128, 1, 2, 3, 4, 5, 128,
128, 0, 1, 2, 3, 4, 5, 128,
0, 1, 2, 3, 4, 5, 6, 128,
128, 128, 128, 128, 128, 128, 128, 0,
0, 128, 128, 128, 128, 128, 128, 1,
128, 0, 128, 128, 128, 128, 128, 1,
0, 1, 128, 128, 128, 128, 128, 2,
128, 128, 0, 128, 128, 128, 128, 1,
0, 128, 1, 128, 128, 128, 128, 2,
128, 0, 1, 128, 128, 128, 128, 2,
0, 1, 2, 128, 128, 128, 128, 3,
128, 128, 128, 0, 128, 128, 128, 1,
0, 128, 128, 1, 128, 128, 128, 2,
128, 0, 128, 1, 128, 128, 128, 2,
0, 1, 128, 2, 128, 128, 128, 3,
128, 128, 0, 1, 128, 128, 128, 2,
0, 128, 1, 2, 128, 128, 128, 3,
128, 0, 1, 2, 128, 128, 128, 3,
0, 1, 2, 3, 128, 128, 128, 4,
128, 128, 128, 128, 0, 128, 128, 1,
0, 128, 128, 128, 1, 128, 128, 2,
128, 0, 128, 128, 1, 128, 128, 2,
0, 1, 128, 128, 2, 128, 128, 3,
128, 128, 0, 128, 1, 128, 128, 2,
0, 128, 1, 128, 2, 128, 128, 3,
128, 0, 1, 128, 2, 128, 128, 3,
0, 1, 2, 128, 3, 128, 128, 4,
128, 128, 128, 0, 1, 128, 128, 2,
0, 128, 128, 1, 2, 128, 128, 3,
128, 0, 128, 1, 2, 128, 128, 3,
0, 1, 128, 2, 3, 128, 128, 4,
128, 128, 0, 1, 2, 128, 128, 3,
0, 128, 1, 2, 3, 128, 128, 4,
128, 0, 1, 2, 3, 128, 128, 4,
0, 1, 2, 3, 4, 128, 128, 5,
128, 128, 128, 128, 128, 0, 128, 1,
0, 128, 128, 128, 128, 1, 128, 2,
128, 0, 128, 128, 128, 1, 128, 2,
0, 1, 128, 128, 128, 2, 128, 3,
128, 128, 0, 128, 128, 1, 128, 2,
0, 128, 1, 128, 128, 2, 128, 3,
128, 0, 1, 128, 128, 2, 128, 3,
0, 1, 2, 128, 128, 3, 128, 4,
128, 128, 128, 0, 128, 1, 128, 2,
0, 128, 128, 1, 128, 2, 128, 3,
128, 0, 128, 1, 128, 2, 128, 3,
0, 1, 128, 2, 128, 3, 128, 4,
128, 128, 0, 1, 128, 2, 128, 3,
0, 128, 1, 2, 128, 3, 128, 4,
128, 0, 1, 2, 128, 3, 128, 4,
0, 1, 2, 3, 128, 4, 128, 5,
128, 128, 128, 128, 0, 1, 128, 2,
0, 128, 128, 128, 1, 2, 128, 3,
128, 0, 128, 128, 1, 2, 128, 3,
0, 1, 128, 128, 2, 3, 128, 4,
128, 128, 0, 128, 1, 2, 128, 3,
0, 128, 1, 128, 2, 3, 128, 4,
128, 0, 1, 128, 2, 3, 128, 4,
0, 1, 2, 128, 3, 4, 128, 5,
128, 128, 128, 0, 1, 2, 128, 3,
0, 128, 128, 1, 2, 3, 128, 4,
128, 0, 128, 1, 2, 3, 128, 4,
0, 1, 128, 2, 3, 4, 128, 5,
128, 128, 0, 1, 2, 3, 128, 4,
0, 128, 1, 2, 3, 4, 128, 5,
128, 0, 1, 2, 3, 4, 128, 5,
0, 1, 2, 3, 4, 5, 128, 6,
128, 128, 128, 128, 128, 128, 0, 1,
0, 128, 128, 128, 128, 128, 1, 2,
128, 0, 128, 128, 128, 128, 1, 2,
0, 1, 128, 128, 128, 128, 2, 3,
128, 128, 0, 128, 128, 128, 1, 2,
0, 128, 1, 128, 128, 128, 2, 3,
128, 0, 1, 128, 128, 128, 2, 3,
0, 1, 2, 128, 128, 128, 3, 4,
128, 128, 128, 0, 128, 128, 1, 2,
0, 128, 128, 1, 128, 128, 2, 3,
128, 0, 128, 1, 128, 128, 2, 3,
0, 1, 128, 2, 128, 128, 3, 4,
128, 128, 0, 1, 128, 128, 2, 3,
0, 128, 1, 2, 128, 128, 3, 4,
128, 0, 1, 2, 128, 128, 3, 4,
0, 1, 2, 3, 128, 128, 4, 5,
128, 128, 128, 128, 0, 128, 1, 2,
0, 128, 128, 128, 1, 128, 2, 3,
128, 0, 128, 128, 1, 128, 2, 3,
0, 1, 128, 128, 2, 128, 3, 4,
128, 128, 0, 128, 1, 128, 2, 3,
0, 128, 1, 128, 2, 128, 3, 4,
128, 0, 1, 128, 2, 128, 3, 4,
0, 1, 2, 128, 3, 128, 4, 5,
128, 128, 128, 0, 1, 128, 2, 3,
0, 128, 128, 1, 2, 128, 3, 4,
128, 0, 128, 1, 2, 128, 3, 4,
0, 1, 128, 2, 3, 128, 4, 5,
128, 128, 0, 1, 2, 128, 3, 4,
0, 128, 1, 2, 3, 128, 4, 5,
128, 0, 1, 2, 3, 128, 4, 5,
0, 1, 2, 3, 4, 128, 5, 6,
128, 128, 128, 128, 128, 0, 1, 2,
0, 128, 128, 128, 128, 1, 2, 3,
128, 0, 128, 128, 128, 1, 2, 3,
0, 1, 128, 128, 128, 2, 3, 4,
128, 128, 0, 128, 128, 1, 2, 3,
0, 128, 1, 128, 128, 2, 3, 4,
128, 0, 1, 128, 128, 2, 3, 4,
0, 1, 2, 128, 128, 3, 4, 5,
128, 128, 128, 0, 128, 1, 2, 3,
0, 128, 128, 1, 128, 2, 3, 4,
128, 0, 128, 1, 128, 2, 3, 4,
0, 1, 128, 2, 128, 3, 4, 5,
128, 128, 0, 1, 128, 2, 3, 4,
0, 128, 1, 2, 128, 3, 4, 5,
128, 0, 1, 2, 128, 3, 4, 5,
0, 1, 2, 3, 128, 4, 5, 6,
128, 128, 128, 128, 0, 1, 2, 3,
0, 128, 128, 128, 1, 2, 3, 4,
128, 0, 128, 128, 1, 2, 3, 4,
0, 1, 128, 128, 2, 3, 4, 5,
128, 128, 0, 128, 1, 2, 3, 4,
0, 128, 1, 128, 2, 3, 4, 5,
128, 0, 1, 128, 2, 3, 4, 5,
0, 1, 2, 128, 3, 4, 5, 6,
128, 128, 128, 0, 1, 2, 3, 4,
0, 128, 128, 1, 2, 3, 4, 5,
128, 0, 128, 1, 2, 3, 4, 5,
0, 1, 128, 2, 3, 4, 5, 6,
128, 128, 0, 1, 2, 3, 4, 5,
0, 128, 1, 2, 3, 4, 5, 6,
128, 0, 1, 2, 3, 4, 5, 6,
0, 1, 2, 3, 4, 5, 6, 7};
return LoadU(du8, table + mask_bits * 8);
}
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (1)>* = nullptr, hwy::EnableIf<N * sizeof(T) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Expand(Vec128<T, N> v, Mask128<T, N> mask) {
const DFromV<decltype(v)> d;
const uint64_t mask_bits = detail::BitsFromMask(mask);
const Vec128<uint8_t, N> indices =
detail::IndicesForExpandFromBits<N>(mask_bits);
return BitCast(d, TableLookupBytesOr0(v, indices));
}
template <typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Expand(Vec128<T> v, Mask128<T> mask) {
const Full128<T> d;
const RebindToUnsigned<decltype(d)> du;
const Half<decltype(du)> duh;
const Vec128<uint8_t> vu = BitCast(du, v);
const uint64_t mask_bits = detail::BitsFromMask(mask);
const uint64_t maskL = mask_bits & 0xFF;
const uint64_t maskH = mask_bits >> 8;
alignas(16) static constexpr uint8_t iota[32] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128, 128, 128};
const VFromD<decltype(du)> shift = LoadU(du, iota + PopCount(maskL));
const VFromD<decltype(duh)> vL = LowerHalf(duh, vu);
const VFromD<decltype(duh)> vH =
LowerHalf(duh, TableLookupBytesOr0(vu, shift));
const VFromD<decltype(duh)> idxL = detail::IndicesForExpandFromBits<8>(maskL);
const VFromD<decltype(duh)> idxH = detail::IndicesForExpandFromBits<8>(maskH);
const VFromD<decltype(duh)> expandL = TableLookupBytesOr0(vL, idxL);
const VFromD<decltype(duh)> expandH = TableLookupBytesOr0(vH, idxH);
return BitCast(d, Combine(du, expandH, expandL));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Expand(Vec128<T, N> v, Mask128<T, N> mask) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const Rebind<uint8_t, decltype(d)> du8;
const uint64_t mask_bits = detail::BitsFromMask(mask);
alignas(16) static constexpr uint8_t table[2048] = {
128, 128, 128, 128, 128, 128, 128, 128,
0, 128, 128, 128, 128, 128, 128, 128,
128, 0, 128, 128, 128, 128, 128, 128,
0, 2, 128, 128, 128, 128, 128, 128,
128, 128, 0, 128, 128, 128, 128, 128,
0, 128, 2, 128, 128, 128, 128, 128,
128, 0, 2, 128, 128, 128, 128, 128,
0, 2, 4, 128, 128, 128, 128, 128,
128, 128, 128, 0, 128, 128, 128, 128,
0, 128, 128, 2, 128, 128, 128, 128,
128, 0, 128, 2, 128, 128, 128, 128,
0, 2, 128, 4, 128, 128, 128, 128,
128, 128, 0, 2, 128, 128, 128, 128,
0, 128, 2, 4, 128, 128, 128, 128,
128, 0, 2, 4, 128, 128, 128, 128,
0, 2, 4, 6, 128, 128, 128, 128,
128, 128, 128, 128, 0, 128, 128, 128,
0, 128, 128, 128, 2, 128, 128, 128,
128, 0, 128, 128, 2, 128, 128, 128,
0, 2, 128, 128, 4, 128, 128, 128,
128, 128, 0, 128, 2, 128, 128, 128,
0, 128, 2, 128, 4, 128, 128, 128,
128, 0, 2, 128, 4, 128, 128, 128,
0, 2, 4, 128, 6, 128, 128, 128,
128, 128, 128, 0, 2, 128, 128, 128,
0, 128, 128, 2, 4, 128, 128, 128,
128, 0, 128, 2, 4, 128, 128, 128,
0, 2, 128, 4, 6, 128, 128, 128,
128, 128, 0, 2, 4, 128, 128, 128,
0, 128, 2, 4, 6, 128, 128, 128,
128, 0, 2, 4, 6, 128, 128, 128,
0, 2, 4, 6, 8, 128, 128, 128,
128, 128, 128, 128, 128, 0, 128, 128,
0, 128, 128, 128, 128, 2, 128, 128,
128, 0, 128, 128, 128, 2, 128, 128,
0, 2, 128, 128, 128, 4, 128, 128,
128, 128, 0, 128, 128, 2, 128, 128,
0, 128, 2, 128, 128, 4, 128, 128,
128, 0, 2, 128, 128, 4, 128, 128,
0, 2, 4, 128, 128, 6, 128, 128,
128, 128, 128, 0, 128, 2, 128, 128,
0, 128, 128, 2, 128, 4, 128, 128,
128, 0, 128, 2, 128, 4, 128, 128,
0, 2, 128, 4, 128, 6, 128, 128,
128, 128, 0, 2, 128, 4, 128, 128,
0, 128, 2, 4, 128, 6, 128, 128,
128, 0, 2, 4, 128, 6, 128, 128,
0, 2, 4, 6, 128, 8, 128, 128,
128, 128, 128, 128, 0, 2, 128, 128,
0, 128, 128, 128, 2, 4, 128, 128,
128, 0, 128, 128, 2, 4, 128, 128,
0, 2, 128, 128, 4, 6, 128, 128,
128, 128, 0, 128, 2, 4, 128, 128,
0, 128, 2, 128, 4, 6, 128, 128,
128, 0, 2, 128, 4, 6, 128, 128,
0, 2, 4, 128, 6, 8, 128, 128,
128, 128, 128, 0, 2, 4, 128, 128,
0, 128, 128, 2, 4, 6, 128, 128,
128, 0, 128, 2, 4, 6, 128, 128,
0, 2, 128, 4, 6, 8, 128, 128,
128, 128, 0, 2, 4, 6, 128, 128,
0, 128, 2, 4, 6, 8, 128, 128,
128, 0, 2, 4, 6, 8, 128, 128,
0, 2, 4, 6, 8, 10, 128, 128,
128, 128, 128, 128, 128, 128, 0, 128,
0, 128, 128, 128, 128, 128, 2, 128,
128, 0, 128, 128, 128, 128, 2, 128,
0, 2, 128, 128, 128, 128, 4, 128,
128, 128, 0, 128, 128, 128, 2, 128,
0, 128, 2, 128, 128, 128, 4, 128,
128, 0, 2, 128, 128, 128, 4, 128,
0, 2, 4, 128, 128, 128, 6, 128,
128, 128, 128, 0, 128, 128, 2, 128,
0, 128, 128, 2, 128, 128, 4, 128,
128, 0, 128, 2, 128, 128, 4, 128,
0, 2, 128, 4, 128, 128, 6, 128,
128, 128, 0, 2, 128, 128, 4, 128,
0, 128, 2, 4, 128, 128, 6, 128,
128, 0, 2, 4, 128, 128, 6, 128,
0, 2, 4, 6, 128, 128, 8, 128,
128, 128, 128, 128, 0, 128, 2, 128,
0, 128, 128, 128, 2, 128, 4, 128,
128, 0, 128, 128, 2, 128, 4, 128,
0, 2, 128, 128, 4, 128, 6, 128,
128, 128, 0, 128, 2, 128, 4, 128,
0, 128, 2, 128, 4, 128, 6, 128,
128, 0, 2, 128, 4, 128, 6, 128,
0, 2, 4, 128, 6, 128, 8, 128,
128, 128, 128, 0, 2, 128, 4, 128,
0, 128, 128, 2, 4, 128, 6, 128,
128, 0, 128, 2, 4, 128, 6, 128,
0, 2, 128, 4, 6, 128, 8, 128,
128, 128, 0, 2, 4, 128, 6, 128,
0, 128, 2, 4, 6, 128, 8, 128,
128, 0, 2, 4, 6, 128, 8, 128,
0, 2, 4, 6, 8, 128, 10, 128,
128, 128, 128, 128, 128, 0, 2, 128,
0, 128, 128, 128, 128, 2, 4, 128,
128, 0, 128, 128, 128, 2, 4, 128,
0, 2, 128, 128, 128, 4, 6, 128,
128, 128, 0, 128, 128, 2, 4, 128,
0, 128, 2, 128, 128, 4, 6, 128,
128, 0, 2, 128, 128, 4, 6, 128,
0, 2, 4, 128, 128, 6, 8, 128,
128, 128, 128, 0, 128, 2, 4, 128,
0, 128, 128, 2, 128, 4, 6, 128,
128, 0, 128, 2, 128, 4, 6, 128,
0, 2, 128, 4, 128, 6, 8, 128,
128, 128, 0, 2, 128, 4, 6, 128,
0, 128, 2, 4, 128, 6, 8, 128,
128, 0, 2, 4, 128, 6, 8, 128,
0, 2, 4, 6, 128, 8, 10, 128,
128, 128, 128, 128, 0, 2, 4, 128,
0, 128, 128, 128, 2, 4, 6, 128,
128, 0, 128, 128, 2, 4, 6, 128,
0, 2, 128, 128, 4, 6, 8, 128,
128, 128, 0, 128, 2, 4, 6, 128,
0, 128, 2, 128, 4, 6, 8, 128,
128, 0, 2, 128, 4, 6, 8, 128,
0, 2, 4, 128, 6, 8, 10, 128,
128, 128, 128, 0, 2, 4, 6, 128,
0, 128, 128, 2, 4, 6, 8, 128,
128, 0, 128, 2, 4, 6, 8, 128,
0, 2, 128, 4, 6, 8, 10, 128,
128, 128, 0, 2, 4, 6, 8, 128,
0, 128, 2, 4, 6, 8, 10, 128,
128, 0, 2, 4, 6, 8, 10, 128,
0, 2, 4, 6, 8, 10, 12, 128,
128, 128, 128, 128, 128, 128, 128, 0,
0, 128, 128, 128, 128, 128, 128, 2,
128, 0, 128, 128, 128, 128, 128, 2,
0, 2, 128, 128, 128, 128, 128, 4,
128, 128, 0, 128, 128, 128, 128, 2,
0, 128, 2, 128, 128, 128, 128, 4,
128, 0, 2, 128, 128, 128, 128, 4,
0, 2, 4, 128, 128, 128, 128, 6,
128, 128, 128, 0, 128, 128, 128, 2,
0, 128, 128, 2, 128, 128, 128, 4,
128, 0, 128, 2, 128, 128, 128, 4,
0, 2, 128, 4, 128, 128, 128, 6,
128, 128, 0, 2, 128, 128, 128, 4,
0, 128, 2, 4, 128, 128, 128, 6,
128, 0, 2, 4, 128, 128, 128, 6,
0, 2, 4, 6, 128, 128, 128, 8,
128, 128, 128, 128, 0, 128, 128, 2,
0, 128, 128, 128, 2, 128, 128, 4,
128, 0, 128, 128, 2, 128, 128, 4,
0, 2, 128, 128, 4, 128, 128, 6,
128, 128, 0, 128, 2, 128, 128, 4,
0, 128, 2, 128, 4, 128, 128, 6,
128, 0, 2, 128, 4, 128, 128, 6,
0, 2, 4, 128, 6, 128, 128, 8,
128, 128, 128, 0, 2, 128, 128, 4,
0, 128, 128, 2, 4, 128, 128, 6,
128, 0, 128, 2, 4, 128, 128, 6,
0, 2, 128, 4, 6, 128, 128, 8,
128, 128, 0, 2, 4, 128, 128, 6,
0, 128, 2, 4, 6, 128, 128, 8,
128, 0, 2, 4, 6, 128, 128, 8,
0, 2, 4, 6, 8, 128, 128, 10,
128, 128, 128, 128, 128, 0, 128, 2,
0, 128, 128, 128, 128, 2, 128, 4,
128, 0, 128, 128, 128, 2, 128, 4,
0, 2, 128, 128, 128, 4, 128, 6,
128, 128, 0, 128, 128, 2, 128, 4,
0, 128, 2, 128, 128, 4, 128, 6,
128, 0, 2, 128, 128, 4, 128, 6,
0, 2, 4, 128, 128, 6, 128, 8,
128, 128, 128, 0, 128, 2, 128, 4,
0, 128, 128, 2, 128, 4, 128, 6,
128, 0, 128, 2, 128, 4, 128, 6,
0, 2, 128, 4, 128, 6, 128, 8,
128, 128, 0, 2, 128, 4, 128, 6,
0, 128, 2, 4, 128, 6, 128, 8,
128, 0, 2, 4, 128, 6, 128, 8,
0, 2, 4, 6, 128, 8, 128, 10,
128, 128, 128, 128, 0, 2, 128, 4,
0, 128, 128, 128, 2, 4, 128, 6,
128, 0, 128, 128, 2, 4, 128, 6,
0, 2, 128, 128, 4, 6, 128, 8,
128, 128, 0, 128, 2, 4, 128, 6,
0, 128, 2, 128, 4, 6, 128, 8,
128, 0, 2, 128, 4, 6, 128, 8,
0, 2, 4, 128, 6, 8, 128, 10,
128, 128, 128, 0, 2, 4, 128, 6,
0, 128, 128, 2, 4, 6, 128, 8,
128, 0, 128, 2, 4, 6, 128, 8,
0, 2, 128, 4, 6, 8, 128, 10,
128, 128, 0, 2, 4, 6, 128, 8,
0, 128, 2, 4, 6, 8, 128, 10,
128, 0, 2, 4, 6, 8, 128, 10,
0, 2, 4, 6, 8, 10, 128, 12,
128, 128, 128, 128, 128, 128, 0, 2,
0, 128, 128, 128, 128, 128, 2, 4,
128, 0, 128, 128, 128, 128, 2, 4,
0, 2, 128, 128, 128, 128, 4, 6,
128, 128, 0, 128, 128, 128, 2, 4,
0, 128, 2, 128, 128, 128, 4, 6,
128, 0, 2, 128, 128, 128, 4, 6,
0, 2, 4, 128, 128, 128, 6, 8,
128, 128, 128, 0, 128, 128, 2, 4,
0, 128, 128, 2, 128, 128, 4, 6,
128, 0, 128, 2, 128, 128, 4, 6,
0, 2, 128, 4, 128, 128, 6, 8,
128, 128, 0, 2, 128, 128, 4, 6,
0, 128, 2, 4, 128, 128, 6, 8,
128, 0, 2, 4, 128, 128, 6, 8,
0, 2, 4, 6, 128, 128, 8, 10,
128, 128, 128, 128, 0, 128, 2, 4,
0, 128, 128, 128, 2, 128, 4, 6,
128, 0, 128, 128, 2, 128, 4, 6,
0, 2, 128, 128, 4, 128, 6, 8,
128, 128, 0, 128, 2, 128, 4, 6,
0, 128, 2, 128, 4, 128, 6, 8,
128, 0, 2, 128, 4, 128, 6, 8,
0, 2, 4, 128, 6, 128, 8, 10,
128, 128, 128, 0, 2, 128, 4, 6,
0, 128, 128, 2, 4, 128, 6, 8,
128, 0, 128, 2, 4, 128, 6, 8,
0, 2, 128, 4, 6, 128, 8, 10,
128, 128, 0, 2, 4, 128, 6, 8,
0, 128, 2, 4, 6, 128, 8, 10,
128, 0, 2, 4, 6, 128, 8, 10,
0, 2, 4, 6, 8, 128, 10, 12,
128, 128, 128, 128, 128, 0, 2, 4,
0, 128, 128, 128, 128, 2, 4, 6,
128, 0, 128, 128, 128, 2, 4, 6,
0, 2, 128, 128, 128, 4, 6, 8,
128, 128, 0, 128, 128, 2, 4, 6,
0, 128, 2, 128, 128, 4, 6, 8,
128, 0, 2, 128, 128, 4, 6, 8,
0, 2, 4, 128, 128, 6, 8, 10,
128, 128, 128, 0, 128, 2, 4, 6,
0, 128, 128, 2, 128, 4, 6, 8,
128, 0, 128, 2, 128, 4, 6, 8,
0, 2, 128, 4, 128, 6, 8, 10,
128, 128, 0, 2, 128, 4, 6, 8,
0, 128, 2, 4, 128, 6, 8, 10,
128, 0, 2, 4, 128, 6, 8, 10,
0, 2, 4, 6, 128, 8, 10, 12,
128, 128, 128, 128, 0, 2, 4, 6,
0, 128, 128, 128, 2, 4, 6, 8,
128, 0, 128, 128, 2, 4, 6, 8,
0, 2, 128, 128, 4, 6, 8, 10,
128, 128, 0, 128, 2, 4, 6, 8,
0, 128, 2, 128, 4, 6, 8, 10,
128, 0, 2, 128, 4, 6, 8, 10,
0, 2, 4, 128, 6, 8, 10, 12,
128, 128, 128, 0, 2, 4, 6, 8,
0, 128, 128, 2, 4, 6, 8, 10,
128, 0, 128, 2, 4, 6, 8, 10,
0, 2, 128, 4, 6, 8, 10, 12,
128, 128, 0, 2, 4, 6, 8, 10,
0, 128, 2, 4, 6, 8, 10, 12,
128, 0, 2, 4, 6, 8, 10, 12,
0, 2, 4, 6, 8, 10, 12, 14};
const Twice<decltype(du8)> du8x2;
const Vec128<uint8_t, 2 * N> indices8 =
ZeroExtendVector(du8x2, Load(du8, table + mask_bits * 8));
const Vec128<uint16_t, N> indices16 =
BitCast(du, InterleaveLower(du8x2, indices8, indices8));
const Vec128<uint16_t, N> byte_indices = Add(indices16, Set(du, 0x0100));
return BitCast(d, TableLookupBytesOr0(v, byte_indices));
}
template <typename T, size_t N, hwy::EnableIf<sizeof(T) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Expand(Vec128<T, N> v, Mask128<T, N> mask) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const uint64_t mask_bits = detail::BitsFromMask(mask);
alignas(16) static constexpr uint32_t packed_array[16] = {
0x0000ffff, 0x0000fff0, 0x0000ff0f, 0x0000ff10, 0x0000f0ff, 0x0000f1f0,
0x0000f10f, 0x0000f210, 0x00000fff, 0x00001ff0, 0x00001f0f, 0x00002f10,
0x000010ff, 0x000021f0, 0x0000210f, 0x00003210};
const Vec128<uint32_t, N> packed = Set(du, packed_array[mask_bits]);
alignas(16) static constexpr uint32_t shifts[4] = {0, 4, 8, 12};
Vec128<uint32_t, N> indices = packed >> Load(du, shifts);
indices = And(indices, Set(du, N - 1));
const Vec128<uint32_t, N> expand =
TableLookupLanes(BitCast(du, v), IndicesFromVec(du, indices));
return IfThenElseZero(mask, BitCast(d, expand));
}
template <typename T, hwy::EnableIf<sizeof(T) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Expand(Vec128<T> v, Mask128<T> mask) {
return IfThenElseZero(mask, Compress(v, mask));
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, 1> Expand(Vec128<T, 1> v, Mask128<T, 1> mask) {
return IfThenElseZero(mask, v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> LoadExpand(MFromD<D> mask, D d,
const TFromD<D>* __restrict__ unaligned) {
return Expand(LoadU(d, unaligned), mask);
}
template <class D>
using IndicesFromD = decltype(IndicesFromVec(D(), Zero(RebindToUnsigned<D>())));
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TwoTablesLookupLanes(D , VFromD<D> a, VFromD<D> b,
IndicesFromD<D> idx) {
return TwoTablesLookupLanes(a, b, idx);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse2(D d, VFromD<D> v) {
alignas(16) static constexpr TFromD<D> kShuffle[16] = {
1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14};
return TableLookupBytes(v, LoadDup128(d, kShuffle));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse4(D d, VFromD<D> v) {
alignas(16) static constexpr uint8_t kShuffle[16] = {
3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12};
const Repartition<uint8_t, decltype(d)> du8;
return TableLookupBytes(v, BitCast(d, LoadDup128(du8, kShuffle)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse8(D d, VFromD<D> v) {
alignas(16) static constexpr uint8_t kShuffle[16] = {
7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8};
const Repartition<uint8_t, decltype(d)> du8;
return TableLookupBytes(v, BitCast(d, LoadDup128(du8, kShuffle)));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseLaneBytes(V v) {
const DFromV<V> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, Reverse2(du8, BitCast(du8, v)));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseLaneBytes(V v) {
const DFromV<V> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, Reverse4(du8, BitCast(du8, v)));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseLaneBytes(V v) {
const DFromV<V> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, Reverse8(du8, BitCast(du8, v)));
}
namespace detail {
template <int kShiftAmt, int kShrResultMask, class V,
hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromD<DFromV<V> >) > 2 - 1)>* = nullptr>
inline __attribute__((always_inline)) V UI8ReverseBitsStep(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const Repartition<uint16_t, decltype(d)> d_shift;
const auto v_to_shift = BitCast(d_shift, v);
const auto shl_result = BitCast(d, ShiftLeft<kShiftAmt>(v_to_shift));
const auto shr_result = BitCast(d, ShiftRight<kShiftAmt>(v_to_shift));
const auto shr_result_mask =
BitCast(d, Set(du, static_cast<uint8_t>(kShrResultMask)));
return Or(And(shr_result, shr_result_mask),
AndNot(shr_result_mask, shl_result));
}
template <int kShiftAmt, int kShrResultMask, class V,
hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromD<DFromV<V> >) == 1>* = nullptr>
inline __attribute__((always_inline)) V UI8ReverseBitsStep(V v) {
return V{UI8ReverseBitsStep<kShiftAmt, kShrResultMask>(Vec128<uint8_t>{v.raw})
.raw};
}
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseBits(V v) {
auto result = detail::UI8ReverseBitsStep<1, 0x55>(v);
result = detail::UI8ReverseBitsStep<2, 0x33>(result);
result = detail::UI8ReverseBitsStep<4, 0x0F>(result);
return result;
}
template <class V, hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 2) | (1 << 4) | (1 << 8))) != 0>* = nullptr,
hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseBits(V v) {
const DFromV<decltype(v)> d;
const Repartition<uint8_t, decltype(d)> du8;
return ReverseLaneBytes(BitCast(d, ReverseBits(BitCast(du8, v))));
}
namespace detail {
template <class V>
inline __attribute__((always_inline)) V Per2LaneBlockShuffle(hwy::SizeTag<0> , V v) {
return DupEven(v);
}
template <class V>
inline __attribute__((always_inline)) V Per2LaneBlockShuffle(hwy::SizeTag<1> , V v) {
const DFromV<decltype(v)> d;
return Reverse2(d, v);
}
template <class V>
inline __attribute__((always_inline)) V Per2LaneBlockShuffle(hwy::SizeTag<2> , V v) {
return v;
}
template <class V>
inline __attribute__((always_inline)) V Per2LaneBlockShuffle(hwy::SizeTag<3> , V v) {
return DupOdd(v);
}
inline __attribute__((always_inline)) uint32_t U8x4Per4LaneBlkIndices(const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
return static_cast<uint32_t>((idx3 << 24) | (idx2 << 16) | (idx1 << 8) |
idx0);
}
template <class D>
inline __attribute__((always_inline)) Vec<D> TblLookupPer4LaneBlkU8IdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const Repartition<uint32_t, D> du32;
return ResizeBitCast(
d, Set(du32, U8x4Per4LaneBlkIndices(idx3, idx2, idx1, idx0)));
}
template <class V, hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2) | (1 << 4))) != 0>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlkShufDoTblLookup(V v, V idx) {
const DFromV<decltype(v)> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, TableLookupBytes(BitCast(du8, v), BitCast(du8, idx)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
inline __attribute__((always_inline)) Vec<D> TblLookupPer4LaneBlkShufIdx(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const Repartition<uint32_t, decltype(d)> du32;
const uint32_t idx3210 = U8x4Per4LaneBlkIndices(idx3, idx2, idx1, idx0);
const auto v_byte_idx = Per4LaneBlkShufDupSet4xU32(
du32, static_cast<uint32_t>(idx3210 + 0x0C0C0C0C),
static_cast<uint32_t>(idx3210 + 0x08080808),
static_cast<uint32_t>(idx3210 + 0x04040404),
static_cast<uint32_t>(idx3210));
return ResizeBitCast(d, v_byte_idx);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
inline __attribute__((always_inline)) Vec<D> TblLookupPer4LaneBlkShufIdx(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const Repartition<uint32_t, decltype(d)> du32;
const uint32_t idx10 = static_cast<uint32_t>((idx1 << 16) | idx0);
const uint32_t idx32 = static_cast<uint32_t>((idx3 << 16) | idx2);
constexpr uint32_t kLaneByteOffsets{0x01000100};
constexpr uint32_t kHiLaneByteOffsets{kLaneByteOffsets + 0x08080808u};
const auto v_byte_idx = Per4LaneBlkShufDupSet4xU32(
du32, static_cast<uint32_t>(idx32 * 0x0202u + kHiLaneByteOffsets),
static_cast<uint32_t>(idx10 * 0x0202u + kHiLaneByteOffsets),
static_cast<uint32_t>(idx32 * 0x0202u + kLaneByteOffsets),
static_cast<uint32_t>(idx10 * 0x0202u + kLaneByteOffsets));
return ResizeBitCast(d, v_byte_idx);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
inline __attribute__((always_inline)) Vec<D> TblLookupPer4LaneBlkShufIdx(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const Repartition<uint32_t, decltype(d)> du32;
constexpr uint32_t kLaneByteOffsets{0x03020100};
const auto v_byte_idx = Per4LaneBlkShufDupSet4xU32(
du32, static_cast<uint32_t>(idx3 * 0x04040404u + kLaneByteOffsets),
static_cast<uint32_t>(idx2 * 0x04040404u + kLaneByteOffsets),
static_cast<uint32_t>(idx1 * 0x04040404u + kLaneByteOffsets),
static_cast<uint32_t>(idx0 * 0x04040404u + kLaneByteOffsets));
return ResizeBitCast(d, v_byte_idx);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> TblLookupPer4LaneBlkIdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
return TblLookupPer4LaneBlkU8IdxInBlk(d, idx3, idx2, idx1, idx0);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> TblLookupPer4LaneBlkIdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const uint16_t u16_idx0 = static_cast<uint16_t>(idx0);
const uint16_t u16_idx1 = static_cast<uint16_t>(idx1);
const uint16_t u16_idx2 = static_cast<uint16_t>(idx2);
const uint16_t u16_idx3 = static_cast<uint16_t>(idx3);
alignas(16)
const uint16_t indices[8] = {u16_idx0, u16_idx1, u16_idx2, u16_idx3,
u16_idx0, u16_idx1, u16_idx2, u16_idx3};
constexpr size_t kMinLanesToLoad = 8;
constexpr size_t kNumToLoad = ((D::kPrivateLanes) > (kMinLanesToLoad) ? (D::kPrivateLanes) : (kMinLanesToLoad));
const CappedTag<uint16_t, kNumToLoad> d_load;
return ResizeBitCast(d, LoadDup128(d_load, indices));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> TblLookupPer4LaneBlkIdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
return Per4LaneBlkShufDupSet4xU32(d, idx3, idx2, idx1, idx0);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> TblLookupPer4LaneBlkIdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const RebindToUnsigned<decltype(d)> du;
const Rebind<uint32_t, decltype(d)> du32;
return BitCast(d, PromoteTo(du, Per4LaneBlkShufDupSet4xU32(du32, idx3, idx2,
idx1, idx0)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
inline __attribute__((always_inline)) IndicesFromD<D> TblLookupPer4LaneBlkShufIdx(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
auto idx_in_blk = TblLookupPer4LaneBlkIdxInBlk(du, idx3, idx2, idx1, idx0);
constexpr size_t kN = D::kPrivateLanes;
if (kN < 4) {
idx_in_blk = And(idx_in_blk, Set(du, static_cast<TU>(kN - 1)));
}
const auto blk_offsets =
And(Iota(du, TU{0}), Set(du, static_cast<TU>(~TU{3})));
return IndicesFromVec(d, Add(idx_in_blk, blk_offsets));
}
template <class V, hwy::EnableIf<sizeof(TFromD<DFromV<V> >) == (8)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlkShufDoTblLookup(V v, IndicesFromD<DFromV<V>> idx) {
return TableLookupLanes(v, idx);
}
template <class V>
inline __attribute__((always_inline)) V TblLookupPer4LaneBlkShuf(V v, size_t idx3210) {
const DFromV<decltype(v)> d;
const uint32_t idx3 = static_cast<uint32_t>((idx3210 >> 6) & 3);
const uint32_t idx2 = static_cast<uint32_t>((idx3210 >> 4) & 3);
const uint32_t idx1 = static_cast<uint32_t>((idx3210 >> 2) & 3);
const uint32_t idx0 = static_cast<uint32_t>(idx3210 & 3);
const auto idx = TblLookupPer4LaneBlkShufIdx(d, idx3, idx2, idx1, idx0);
return Per4LaneBlkShufDoTblLookup(v, idx);
}
template <size_t kIdx3210, size_t kLaneSize, size_t kVectSize, class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> ,
hwy::SizeTag<kLaneSize> ,
hwy::SizeTag<kVectSize> ,
V v) {
return TblLookupPer4LaneBlkShuf(v, kIdx3210);
}
template <class V>
inline __attribute__((always_inline)) VFromD<RepartitionToWide<DFromV<V>>> Per4LaneBlockShufCastToWide(
hwy::FloatTag , hwy::SizeTag<4> , V v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> dw;
return BitCast(dw, v);
}
template <size_t kLaneSize, class V>
inline __attribute__((always_inline)) VFromD<RepartitionToWide<RebindToUnsigned<DFromV<V>>>>
Per4LaneBlockShufCastToWide(hwy::FloatTag ,
hwy::SizeTag<kLaneSize> , V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const RepartitionToWide<decltype(du)> dw;
return BitCast(dw, v);
}
template <size_t kLaneSize, class V>
inline __attribute__((always_inline)) VFromD<RepartitionToWide<DFromV<V>>> Per4LaneBlockShufCastToWide(
hwy::NonFloatTag ,
hwy::SizeTag<kLaneSize> , V v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> dw;
return BitCast(dw, v);
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x1B> , V v) {
const DFromV<decltype(v)> d;
return Reverse4(d, v);
}
template <class V,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2) | (1 ? (1 << 4) : 0))) != 0>* = nullptr
>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x44> , V v) {
const DFromV<decltype(v)> d;
const auto vw = Per4LaneBlockShufCastToWide(
hwy::IsFloatTag<TFromV<V>>(), hwy::SizeTag<sizeof(TFromV<V>)>(), v);
return BitCast(d, DupEven(vw));
}
template <class V,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2) | (1 ? (1 << 4) : 0))) != 0>* = nullptr
>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x4E> , V v) {
const DFromV<decltype(v)> d;
const auto vw = Per4LaneBlockShufCastToWide(
hwy::IsFloatTag<TFromV<V>>(), hwy::SizeTag<sizeof(TFromV<V>)>(), v);
const DFromV<decltype(vw)> dw;
return BitCast(d, Reverse2(dw, vw));
}
template <class V, hwy::EnableIf<(DFromV<V>::kPrivateLanes == 4)>* = nullptr,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2))) != 0>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x50> , V v) {
const DFromV<decltype(v)> d;
return InterleaveLower(d, v, v);
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (4)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x50> , V v) {
const DFromV<decltype(v)> d;
return InterleaveLower(d, v, v);
}
template <class V, hwy::EnableIf<(DFromV<V>::kPrivateLanes == 4)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x88> , V v) {
const DFromV<decltype(v)> d;
return ConcatEven(d, v, v);
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xA0> , V v) {
return DupEven(v);
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xB1> , V v) {
const DFromV<decltype(v)> d;
return Reverse2(d, v);
}
template <class V, hwy::EnableIf<(DFromV<V>::kPrivateLanes == 4)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xDD> , V v) {
const DFromV<decltype(v)> d;
return ConcatOdd(d, v, v);
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xE4> , V v) {
return v;
}
template <class V,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2) | (1 ? (1 << 4) : 0))) != 0>* = nullptr
>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xEE> , V v) {
const DFromV<decltype(v)> d;
const auto vw = Per4LaneBlockShufCastToWide(
hwy::IsFloatTag<TFromV<V>>(), hwy::SizeTag<sizeof(TFromV<V>)>(), v);
return BitCast(d, DupOdd(vw));
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xF5> , V v) {
return DupOdd(v);
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (4)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xFA> , V v) {
const DFromV<decltype(v)> d;
return InterleaveUpper(d, v, v);
}
template <size_t kIdx3210, class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> idx_3210_tag, V v) {
const DFromV<decltype(v)> d;
return Per4LaneBlockShuffle(idx_3210_tag, hwy::SizeTag<sizeof(TFromV<V>)>(),
hwy::SizeTag<d.MaxBytes()>(), v);
}
}
template <size_t kIdx3, size_t kIdx2, size_t kIdx1, size_t kIdx0, class V,
hwy::EnableIf<(DFromV<V>::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Per4LaneBlockShuffle(V v) {
static_assert(kIdx0 <= 3, "kIdx0 <= 3 must be true");
static_assert(kIdx1 <= 3, "kIdx1 <= 3 must be true");
static_assert(kIdx2 <= 3, "kIdx2 <= 3 must be true");
static_assert(kIdx3 <= 3, "kIdx3 <= 3 must be true");
return v;
}
template <size_t kIdx3, size_t kIdx2, size_t kIdx1, size_t kIdx0, class V,
hwy::EnableIf<(DFromV<V>::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Per4LaneBlockShuffle(V v) {
static_assert(kIdx0 <= 3, "kIdx0 <= 3 must be true");
static_assert(kIdx1 <= 3, "kIdx1 <= 3 must be true");
static_assert(kIdx2 <= 3, "kIdx2 <= 3 must be true");
static_assert(kIdx3 <= 3, "kIdx3 <= 3 must be true");
constexpr bool isReverse2 = (kIdx0 == 1 || kIdx1 == 0) && (kIdx0 != kIdx1);
constexpr size_t kPer2BlkIdx0 = (kIdx0 <= 1) ? kIdx0 : (isReverse2 ? 1 : 0);
constexpr size_t kPer2BlkIdx1 = (kIdx1 <= 1) ? kIdx1 : (isReverse2 ? 0 : 1);
constexpr size_t kIdx10 = (kPer2BlkIdx1 << 1) | kPer2BlkIdx0;
static_assert(kIdx10 <= 3, "kIdx10 <= 3 must be true");
return detail::Per2LaneBlockShuffle(hwy::SizeTag<kIdx10>(), v);
}
template <size_t kIdx3, size_t kIdx2, size_t kIdx1, size_t kIdx0, class V,
hwy::EnableIf<(DFromV<V>::kPrivateLanes > 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Per4LaneBlockShuffle(V v) {
static_assert(kIdx0 <= 3, "kIdx0 <= 3 must be true");
static_assert(kIdx1 <= 3, "kIdx1 <= 3 must be true");
static_assert(kIdx2 <= 3, "kIdx2 <= 3 must be true");
static_assert(kIdx3 <= 3, "kIdx3 <= 3 must be true");
constexpr size_t kIdx3210 =
(kIdx3 << 6) | (kIdx2 << 4) | (kIdx1 << 2) | kIdx0;
return detail::Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210>(), v);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t Blocks(D d) {
return (d.MaxBytes() <= 16) ? 1 : ((Lanes(d) * sizeof(TFromD<D>) + 15) / 16);
}
template <int kBlockIdx, class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V InsertBlock(V , V blk_to_insert) {
static_assert(kBlockIdx == 0, "Invalid block index");
return blk_to_insert;
}
template <int kBlockIdx, class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ExtractBlock(V v) {
static_assert(kBlockIdx == 0, "Invalid block index");
return v;
}
template <int kBlockIdx, class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V BroadcastBlock(V v) {
static_assert(kBlockIdx == 0, "Invalid block index");
return v;
}
template<int kLane, class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V BroadcastLane(V v) {
return Broadcast<kLane>(v);
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Slide1Up(D d, VFromD<D> ) {
return Zero(d);
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Slide1Down(D d, VFromD<D> ) {
return Zero(d);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Slide1Up(D d, VFromD<D> v) {
return ShiftLeftLanes<1>(d, v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Slide1Down(D d, VFromD<D> v) {
return ShiftRightLanes<1>(d, v);
}
template <int kBlocks, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpBlocks(D , VFromD<D> v) {
static_assert(kBlocks == 0, "kBlocks == 0 must be true");
return v;
}
template <int kBlocks, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownBlocks(D , VFromD<D> v) {
static_assert(kBlocks == 0, "kBlocks == 0 must be true");
return v;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Add(V a, V b) {
return a + b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Sub(V a, V b) {
return a - b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Mul(V a, V b) {
return a * b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Div(V a, V b) {
return a / b;
}
template <class V>
V Shl(V a, V b) {
return a << b;
}
template <class V>
V Shr(V a, V b) {
return a >> b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Eq(V a, V b) -> decltype(a == b) {
return a == b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Ne(V a, V b) -> decltype(a == b) {
return a != b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Lt(V a, V b) -> decltype(a == b) {
return a < b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Gt(V a, V b) -> decltype(a == b) {
return a > b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Ge(V a, V b) -> decltype(a == b) {
return a >= b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Le(V a, V b) -> decltype(a == b) {
return a <= b;
}
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC push_options
#pragma GCC target "sse2,ssse3" ",sse4.1,sse4.2" ",pclmul,aes"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSE4 {
template <class D, class V>
inline __attribute__((always_inline)) V Acos(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAcos(const D d, VecArg<V> x) {
return Acos(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Acosh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAcosh(const D d, VecArg<V> x) {
return Acosh(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Asin(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAsin(const D d, VecArg<V> x) {
return Asin(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Asinh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAsinh(const D d, VecArg<V> x) {
return Asinh(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Atan(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAtan(const D d, VecArg<V> x) {
return Atan(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Atanh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAtanh(const D d, VecArg<V> x) {
return Atanh(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Cos(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallCos(const D d, VecArg<V> x) {
return Cos(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Exp(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallExp(const D d, VecArg<V> x) {
return Exp(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Expm1(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallExpm1(const D d, VecArg<V> x) {
return Expm1(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallLog(const D d, VecArg<V> x) {
return Log(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log10(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallLog10(const D d, VecArg<V> x) {
return Log10(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log1p(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallLog1p(const D d, VecArg<V> x) {
return Log1p(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log2(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallLog2(const D d, VecArg<V> x) {
return Log2(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Sin(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallSin(const D d, VecArg<V> x) {
return Sin(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Sinh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallSinh(const D d, VecArg<V> x) {
return Sinh(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Tanh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallTanh(const D d, VecArg<V> x) {
return Tanh(d, x);
}
namespace impl {
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1) {
return MulAdd(c1, x, c0);
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2) {
T x2 = Mul(x, x);
return MulAdd(x2, c2, MulAdd(c1, x, c0));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3) {
T x2 = Mul(x, x);
return MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
return MulAdd(x4, c4, MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
return MulAdd(x4, MulAdd(c5, x, c4),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
return MulAdd(x4, MulAdd(x2, c6, MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
return MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8, c8,
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8, MulAdd(c9, x, c8),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8, MulAdd(x2, c10, MulAdd(c9, x, c8)),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8, MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8)),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(
x8, MulAdd(x4, c12, MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8,
MulAdd(x4, MulAdd(c13, x, c12),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8,
MulAdd(x4, MulAdd(x2, c14, MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14, T c15) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8,
MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14, T c15, T c16) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
T x16 = Mul(x8, x8);
return MulAdd(
x16, c16,
MulAdd(x8,
MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14, T c15, T c16, T c17) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
T x16 = Mul(x8, x8);
return MulAdd(
x16, MulAdd(c17, x, c16),
MulAdd(x8,
MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14, T c15, T c16, T c17,
T c18) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
T x16 = Mul(x8, x8);
return MulAdd(
x16, MulAdd(x2, c18, MulAdd(c17, x, c16)),
MulAdd(x8,
MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))));
}
template <class FloatOrDouble>
struct AsinImpl {};
template <class FloatOrDouble>
struct AtanImpl {};
template <class FloatOrDouble>
struct CosSinImpl {};
template <class FloatOrDouble>
struct ExpImpl {};
template <class FloatOrDouble>
struct LogImpl {};
template <>
struct AsinImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) V AsinPoly(D d, V x2, V ) {
const auto k0 = Set(d, +0.1666677296f);
const auto k1 = Set(d, +0.07495029271f);
const auto k2 = Set(d, +0.04547423869f);
const auto k3 = Set(d, +0.02424046025f);
const auto k4 = Set(d, +0.04197454825f);
return Estrin(x2, k0, k1, k2, k3, k4);
}
};
template <>
struct AsinImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) V AsinPoly(D d, V x2, V ) {
const auto k0 = Set(d, +0.1666666666666497543);
const auto k1 = Set(d, +0.07500000000378581611);
const auto k2 = Set(d, +0.04464285681377102438);
const auto k3 = Set(d, +0.03038195928038132237);
const auto k4 = Set(d, +0.02237176181932048341);
const auto k5 = Set(d, +0.01735956991223614604);
const auto k6 = Set(d, +0.01388715184501609218);
const auto k7 = Set(d, +0.01215360525577377331);
const auto k8 = Set(d, +0.006606077476277170610);
const auto k9 = Set(d, +0.01929045477267910674);
const auto k10 = Set(d, -0.01581918243329996643);
const auto k11 = Set(d, +0.03161587650653934628);
return Estrin(x2, k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11);
}
};
template <>
struct AtanImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) V AtanPoly(D d, V x) {
const auto k0 = Set(d, -0.333331018686294555664062f);
const auto k1 = Set(d, +0.199926957488059997558594f);
const auto k2 = Set(d, -0.142027363181114196777344f);
const auto k3 = Set(d, +0.106347933411598205566406f);
const auto k4 = Set(d, -0.0748900920152664184570312f);
const auto k5 = Set(d, +0.0425049886107444763183594f);
const auto k6 = Set(d, -0.0159569028764963150024414f);
const auto k7 = Set(d, +0.00282363896258175373077393f);
const auto y = Mul(x, x);
return MulAdd(Estrin(y, k0, k1, k2, k3, k4, k5, k6, k7), Mul(y, x), x);
}
};
template <>
struct AtanImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) V AtanPoly(D d, V x) {
const auto k0 = Set(d, -0.333333333333311110369124);
const auto k1 = Set(d, +0.199999999996591265594148);
const auto k2 = Set(d, -0.14285714266771329383765);
const auto k3 = Set(d, +0.111111105648261418443745);
const auto k4 = Set(d, -0.090908995008245008229153);
const auto k5 = Set(d, +0.0769219538311769618355029);
const auto k6 = Set(d, -0.0666573579361080525984562);
const auto k7 = Set(d, +0.0587666392926673580854313);
const auto k8 = Set(d, -0.0523674852303482457616113);
const auto k9 = Set(d, +0.0466667150077840625632675);
const auto k10 = Set(d, -0.0407629191276836500001934);
const auto k11 = Set(d, +0.0337852580001353069993897);
const auto k12 = Set(d, -0.0254517624932312641616861);
const auto k13 = Set(d, +0.016599329773529201970117);
const auto k14 = Set(d, -0.00889896195887655491740809);
const auto k15 = Set(d, +0.00370026744188713119232403);
const auto k16 = Set(d, -0.00110611831486672482563471);
const auto k17 = Set(d, +0.000209850076645816976906797);
const auto k18 = Set(d, -1.88796008463073496563746e-5);
const auto y = Mul(x, x);
return MulAdd(Estrin(y, k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11,
k12, k13, k14, k15, k16, k17, k18),
Mul(y, x), x);
}
};
template <>
struct CosSinImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> ToInt32(D , V x) {
return ConvertTo(Rebind<int32_t, D>(), x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Poly(D d, V x) {
const auto k0 = Set(d, -1.66666597127914428710938e-1f);
const auto k1 = Set(d, +8.33307858556509017944336e-3f);
const auto k2 = Set(d, -1.981069071916863322258e-4f);
const auto k3 = Set(d, +2.6083159809786593541503e-6f);
const auto y = Mul(x, x);
return MulAdd(Estrin(y, k0, k1, k2, k3), Mul(y, x), x);
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V CosReduce(D d, V x, VI32 q) {
const V kHalfPiPart0f = Set(d, -0.5f * 3.140625f);
const V kHalfPiPart1f = Set(d, -0.5f * 0.0009670257568359375f);
const V kHalfPiPart2f = Set(d, -0.5f * 6.2771141529083251953e-7f);
const V kHalfPiPart3f = Set(d, -0.5f * 1.2154201256553420762e-10f);
const V qf = ConvertTo(d, q);
x = MulAdd(qf, kHalfPiPart0f, x);
x = MulAdd(qf, kHalfPiPart1f, x);
x = MulAdd(qf, kHalfPiPart2f, x);
x = MulAdd(qf, kHalfPiPart3f, x);
return x;
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V SinReduce(D d, V x, VI32 q) {
const V kPiPart0f = Set(d, -3.140625f);
const V kPiPart1f = Set(d, -0.0009670257568359375f);
const V kPiPart2f = Set(d, -6.2771141529083251953e-7f);
const V kPiPart3f = Set(d, -1.2154201256553420762e-10f);
const V qf = ConvertTo(d, q);
x = MulAdd(qf, kPiPart0f, x);
x = MulAdd(qf, kPiPart1f, x);
x = MulAdd(qf, kPiPart2f, x);
x = MulAdd(qf, kPiPart3f, x);
return x;
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<Rebind<float, D>> CosSignFromQuadrant(D d, VI32 q) {
const VI32 kTwo = Set(Rebind<int32_t, D>(), 2);
return BitCast(d, ShiftLeft<30>(AndNot(q, kTwo)));
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<Rebind<float, D>> SinSignFromQuadrant(D d, VI32 q) {
const VI32 kOne = Set(Rebind<int32_t, D>(), 1);
return BitCast(d, ShiftLeft<31>(And(q, kOne)));
}
};
template <>
struct CosSinImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> ToInt32(D , V x) {
return DemoteTo(Rebind<int32_t, D>(), x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Poly(D d, V x) {
const auto k0 = Set(d, -0.166666666666666657414808);
const auto k1 = Set(d, +0.00833333333333332974823815);
const auto k2 = Set(d, -0.000198412698412696162806809);
const auto k3 = Set(d, +2.75573192239198747630416e-6);
const auto k4 = Set(d, -2.50521083763502045810755e-8);
const auto k5 = Set(d, +1.60590430605664501629054e-10);
const auto k6 = Set(d, -7.64712219118158833288484e-13);
const auto k7 = Set(d, +2.81009972710863200091251e-15);
const auto k8 = Set(d, -7.97255955009037868891952e-18);
const auto y = Mul(x, x);
return MulAdd(Estrin(y, k0, k1, k2, k3, k4, k5, k6, k7, k8), Mul(y, x), x);
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V CosReduce(D d, V x, VI32 q) {
const V kHalfPiPart0d = Set(d, -0.5 * 3.1415926218032836914);
const V kHalfPiPart1d = Set(d, -0.5 * 3.1786509424591713469e-8);
const V kHalfPiPart2d = Set(d, -0.5 * 1.2246467864107188502e-16);
const V kHalfPiPart3d = Set(d, -0.5 * 1.2736634327021899816e-24);
const V qf = PromoteTo(d, q);
x = MulAdd(qf, kHalfPiPart0d, x);
x = MulAdd(qf, kHalfPiPart1d, x);
x = MulAdd(qf, kHalfPiPart2d, x);
x = MulAdd(qf, kHalfPiPart3d, x);
return x;
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V SinReduce(D d, V x, VI32 q) {
const V kPiPart0d = Set(d, -3.1415926218032836914);
const V kPiPart1d = Set(d, -3.1786509424591713469e-8);
const V kPiPart2d = Set(d, -1.2246467864107188502e-16);
const V kPiPart3d = Set(d, -1.2736634327021899816e-24);
const V qf = PromoteTo(d, q);
x = MulAdd(qf, kPiPart0d, x);
x = MulAdd(qf, kPiPart1d, x);
x = MulAdd(qf, kPiPart2d, x);
x = MulAdd(qf, kPiPart3d, x);
return x;
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<Rebind<double, D>> CosSignFromQuadrant(D d, VI32 q) {
const VI32 kTwo = Set(Rebind<int32_t, D>(), 2);
return BitCast(
d, ShiftLeft<62>(PromoteTo(Rebind<int64_t, D>(), AndNot(q, kTwo))));
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<Rebind<double, D>> SinSignFromQuadrant(D d, VI32 q) {
const VI32 kOne = Set(Rebind<int32_t, D>(), 1);
return BitCast(
d, ShiftLeft<63>(PromoteTo(Rebind<int64_t, D>(), And(q, kOne))));
}
};
template <>
struct ExpImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> ToInt32(D , V x) {
return ConvertTo(Rebind<int32_t, D>(), x);
}
template <class D, class V>
inline __attribute__((always_inline)) V ExpPoly(D d, V x) {
const auto k0 = Set(d, +0.5f);
const auto k1 = Set(d, +0.166666671633720397949219f);
const auto k2 = Set(d, +0.0416664853692054748535156f);
const auto k3 = Set(d, +0.00833336077630519866943359f);
const auto k4 = Set(d, +0.00139304355252534151077271f);
const auto k5 = Set(d, +0.000198527617612853646278381f);
return MulAdd(Estrin(x, k0, k1, k2, k3, k4, k5), Mul(x, x), x);
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<D> Pow2I(D d, VI32 x) {
const Rebind<int32_t, D> di32;
const VI32 kOffset = Set(di32, 0x7F);
return BitCast(d, ShiftLeft<23>(Add(x, kOffset)));
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V LoadExpShortRange(D d, V x, VI32 e) {
const VI32 y = ShiftRight<1>(e);
return Mul(Mul(x, Pow2I(d, y)), Pow2I(d, Sub(e, y)));
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V ExpReduce(D d, V x, VI32 q) {
const V kLn2Part0f = Set(d, -0.693145751953125f);
const V kLn2Part1f = Set(d, -1.428606765330187045e-6f);
const V qf = ConvertTo(d, q);
x = MulAdd(qf, kLn2Part0f, x);
x = MulAdd(qf, kLn2Part1f, x);
return x;
}
};
template <>
struct LogImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> Log2p1NoSubnormal(D , V x) {
const Rebind<int32_t, D> di32;
const Rebind<uint32_t, D> du32;
const auto kBias = Set(di32, 0x7F);
return Sub(BitCast(di32, ShiftRight<23>(BitCast(du32, x))), kBias);
}
template <class D, class V>
inline __attribute__((always_inline)) V LogPoly(D d, V x) {
const V k0 = Set(d, 0.66666662693f);
const V k1 = Set(d, 0.40000972152f);
const V k2 = Set(d, 0.28498786688f);
const V k3 = Set(d, 0.24279078841f);
const V x2 = Mul(x, x);
const V x4 = Mul(x2, x2);
return MulAdd(MulAdd(k2, x4, k0), x2, Mul(MulAdd(k3, x4, k1), x4));
}
};
template <>
struct ExpImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> ToInt32(D , V x) {
return DemoteTo(Rebind<int32_t, D>(), x);
}
template <class D, class V>
inline __attribute__((always_inline)) V ExpPoly(D d, V x) {
const auto k0 = Set(d, +0.5);
const auto k1 = Set(d, +0.166666666666666851703837);
const auto k2 = Set(d, +0.0416666666666665047591422);
const auto k3 = Set(d, +0.00833333333331652721664984);
const auto k4 = Set(d, +0.00138888888889774492207962);
const auto k5 = Set(d, +0.000198412698960509205564975);
const auto k6 = Set(d, +2.4801587159235472998791e-5);
const auto k7 = Set(d, +2.75572362911928827629423e-6);
const auto k8 = Set(d, +2.75573911234900471893338e-7);
const auto k9 = Set(d, +2.51112930892876518610661e-8);
const auto k10 = Set(d, +2.08860621107283687536341e-9);
return MulAdd(Estrin(x, k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10),
Mul(x, x), x);
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<D> Pow2I(D d, VI32 x) {
const Rebind<int32_t, D> di32;
const Rebind<int64_t, D> di64;
const VI32 kOffset = Set(di32, 0x3FF);
return BitCast(d, ShiftLeft<52>(PromoteTo(di64, Add(x, kOffset))));
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V LoadExpShortRange(D d, V x, VI32 e) {
const VI32 y = ShiftRight<1>(e);
return Mul(Mul(x, Pow2I(d, y)), Pow2I(d, Sub(e, y)));
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V ExpReduce(D d, V x, VI32 q) {
const V kLn2Part0d = Set(d, -0.6931471805596629565116018);
const V kLn2Part1d = Set(d, -0.28235290563031577122588448175e-12);
const V qf = PromoteTo(d, q);
x = MulAdd(qf, kLn2Part0d, x);
x = MulAdd(qf, kLn2Part1d, x);
return x;
}
};
template <>
struct LogImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int64_t, D>> Log2p1NoSubnormal(D , V x) {
const Rebind<int64_t, D> di64;
const Rebind<uint64_t, D> du64;
return Sub(BitCast(di64, ShiftRight<52>(BitCast(du64, x))),
Set(di64, 0x3FF));
}
template <class D, class V>
inline __attribute__((always_inline)) V LogPoly(D d, V x) {
const V k0 = Set(d, 0.6666666666666735130);
const V k1 = Set(d, 0.3999999999940941908);
const V k2 = Set(d, 0.2857142874366239149);
const V k3 = Set(d, 0.2222219843214978396);
const V k4 = Set(d, 0.1818357216161805012);
const V k5 = Set(d, 0.1531383769920937332);
const V k6 = Set(d, 0.1479819860511658591);
const V x2 = Mul(x, x);
const V x4 = Mul(x2, x2);
return MulAdd(MulAdd(MulAdd(MulAdd(k6, x4, k4), x4, k2), x4, k0), x2,
(Mul(MulAdd(MulAdd(k5, x4, k3), x4, k1), x4)));
}
};
template <class D, class V, bool kAllowSubnormals = true>
inline __attribute__((always_inline)) V Log(const D d, V x) {
using T = TFromD<D>;
impl::LogImpl<T> impl;
constexpr bool kIsF32 = (sizeof(T) == 4);
const V kLn2Hi = Set(d, kIsF32 ? static_cast<T>(0.69313812256f)
: static_cast<T>(0.693147180369123816490));
const V kLn2Lo = Set(d, kIsF32 ? static_cast<T>(9.0580006145e-6f)
: static_cast<T>(1.90821492927058770002e-10));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kMinNormal = Set(d, kIsF32 ? static_cast<T>(1.175494351e-38f)
: static_cast<T>(2.2250738585072014e-308));
const V kScale = Set(d, kIsF32 ? static_cast<T>(3.355443200e+7f)
: static_cast<T>(1.8014398509481984e+16));
using TI = MakeSigned<T>;
const Rebind<TI, D> di;
using VI = decltype(Zero(di));
const VI kLowerBits = Set(di, kIsF32 ? static_cast<TI>(0x00000000L)
: static_cast<TI>(0xFFFFFFFFLL));
const VI kMagic = Set(di, kIsF32 ? static_cast<TI>(0x3F3504F3L)
: static_cast<TI>(0x3FE6A09E00000000LL));
const VI kExpMask = Set(di, kIsF32 ? static_cast<TI>(0x3F800000L)
: static_cast<TI>(0x3FF0000000000000LL));
const VI kExpScale =
Set(di, kIsF32 ? static_cast<TI>(-25) : static_cast<TI>(-54));
const VI kManMask = Set(di, kIsF32 ? static_cast<TI>(0x7FFFFFL)
: static_cast<TI>(0xFFFFF00000000LL));
VI exp_bits;
V exp;
if (kAllowSubnormals == true) {
const auto is_denormal = Lt(x, kMinNormal);
x = IfThenElse(is_denormal, Mul(x, kScale), x);
exp_bits = Add(BitCast(di, x), Sub(kExpMask, kMagic));
const VI exp_scale =
BitCast(di, IfThenElseZero(is_denormal, BitCast(d, kExpScale)));
exp = ConvertTo(
d, Add(exp_scale, impl.Log2p1NoSubnormal(d, BitCast(d, exp_bits))));
} else {
exp_bits = Add(BitCast(di, x), Sub(kExpMask, kMagic));
exp = ConvertTo(d, impl.Log2p1NoSubnormal(d, BitCast(d, exp_bits)));
}
const V y = Or(And(x, BitCast(d, kLowerBits)),
BitCast(d, Add(And(exp_bits, kManMask), kMagic)));
const V ym1 = Sub(y, kOne);
const V z = Div(ym1, Add(y, kOne));
return MulSub(
exp, kLn2Hi,
Sub(MulSub(z, Sub(ym1, impl.LogPoly(d, z)), Mul(exp, kLn2Lo)), ym1));
}
}
template <class D, class V>
inline __attribute__((always_inline)) V Acos(const D d, V x) {
using T = TFromD<D>;
const V kZero = Zero(d);
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kPi = Set(d, static_cast<T>(+3.14159265358979323846264));
const V kPiOverTwo = Set(d, static_cast<T>(+1.57079632679489661923132169));
const V sign_x = And(SignBit(d), x);
const V abs_x = Xor(x, sign_x);
const auto mask = Lt(abs_x, kHalf);
const V yy =
IfThenElse(mask, Mul(abs_x, abs_x), NegMulAdd(abs_x, kHalf, kHalf));
const V y = IfThenElse(mask, abs_x, Sqrt(yy));
impl::AsinImpl<T> impl;
const V t = Mul(impl.AsinPoly(d, yy, y), Mul(y, yy));
const V t_plus_y = Add(t, y);
const V z =
IfThenElse(mask, Sub(kPiOverTwo, Add(Xor(y, sign_x), Xor(t, sign_x))),
Add(t_plus_y, t_plus_y));
return IfThenElse(Or(mask, Ge(x, kZero)), z, Sub(kPi, z));
}
template <class D, class V>
inline __attribute__((always_inline)) V Acosh(const D d, V x) {
using T = TFromD<D>;
const V kLarge = Set(d, static_cast<T>(268435456.0));
const V kLog2 = Set(d, static_cast<T>(0.693147180559945286227));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kTwo = Set(d, static_cast<T>(+2.0));
const auto is_x_large = Gt(x, kLarge);
const auto is_x_gt_2 = Gt(x, kTwo);
const V x_minus_1 = Sub(x, kOne);
const V y0 = MulSub(kTwo, x, Div(kOne, Add(Sqrt(MulSub(x, x, kOne)), x)));
const V y1 =
Add(Sqrt(MulAdd(x_minus_1, kTwo, Mul(x_minus_1, x_minus_1))), x_minus_1);
const V y2 =
IfThenElse(is_x_gt_2, IfThenElse(is_x_large, x, y0), Add(y1, kOne));
const V z = impl::Log<D, V, false>(d, y2);
const auto is_pole = Eq(y2, kOne);
const auto divisor = Sub(IfThenZeroElse(is_pole, y2), kOne);
return Add(IfThenElse(is_x_gt_2, z,
IfThenElse(is_pole, y1, Div(Mul(z, y1), divisor))),
IfThenElseZero(is_x_large, kLog2));
}
template <class D, class V>
inline __attribute__((always_inline)) V Asin(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kTwo = Set(d, static_cast<T>(+2.0));
const V kPiOverTwo = Set(d, static_cast<T>(+1.57079632679489661923132169));
const V sign_x = And(SignBit(d), x);
const V abs_x = Xor(x, sign_x);
const auto mask = Lt(abs_x, kHalf);
const V yy =
IfThenElse(mask, Mul(abs_x, abs_x), NegMulAdd(abs_x, kHalf, kHalf));
const V y = IfThenElse(mask, abs_x, Sqrt(yy));
impl::AsinImpl<T> impl;
const V z0 = MulAdd(impl.AsinPoly(d, yy, y), Mul(yy, y), y);
const V z1 = NegMulAdd(z0, kTwo, kPiOverTwo);
return Or(IfThenElse(mask, z0, z1), sign_x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Asinh(const D d, V x) {
using T = TFromD<D>;
const V kSmall = Set(d, static_cast<T>(1.0 / 268435456.0));
const V kLarge = Set(d, static_cast<T>(268435456.0));
const V kLog2 = Set(d, static_cast<T>(0.693147180559945286227));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kTwo = Set(d, static_cast<T>(+2.0));
const V sign_x = And(SignBit(d), x);
const V abs_x = Xor(x, sign_x);
const auto is_x_large = Gt(abs_x, kLarge);
const auto is_x_lt_2 = Lt(abs_x, kTwo);
const V x2 = Mul(x, x);
const V sqrt_x2_plus_1 = Sqrt(Add(x2, kOne));
const V y0 = MulAdd(abs_x, kTwo, Div(kOne, Add(sqrt_x2_plus_1, abs_x)));
const V y1 = Add(Div(x2, Add(sqrt_x2_plus_1, kOne)), abs_x);
const V y2 =
IfThenElse(is_x_lt_2, Add(y1, kOne), IfThenElse(is_x_large, abs_x, y0));
const V z = impl::Log<D, V, false>(d, y2);
const auto is_pole = Eq(y2, kOne);
const auto divisor = Sub(IfThenZeroElse(is_pole, y2), kOne);
const auto large = IfThenElse(is_pole, y1, Div(Mul(z, y1), divisor));
const V y = IfThenElse(Lt(abs_x, kSmall), x, large);
return Or(Add(IfThenElse(is_x_lt_2, y, z), IfThenElseZero(is_x_large, kLog2)),
sign_x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Atan(const D d, V x) {
using T = TFromD<D>;
const V kOne = Set(d, static_cast<T>(+1.0));
const V kPiOverTwo = Set(d, static_cast<T>(+1.57079632679489661923132169));
const V sign = And(SignBit(d), x);
const V abs_x = Xor(x, sign);
const auto mask = Gt(abs_x, kOne);
impl::AtanImpl<T> impl;
const auto divisor = IfThenElse(mask, abs_x, kOne);
const V y = impl.AtanPoly(d, IfThenElse(mask, Div(kOne, divisor), abs_x));
return Or(IfThenElse(mask, Sub(kPiOverTwo, y), y), sign);
}
template <class D, class V>
inline __attribute__((always_inline)) V Atanh(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kOne = Set(d, static_cast<T>(+1.0));
const V sign = And(SignBit(d), x);
const V abs_x = Xor(x, sign);
return Mul(Log1p(d, Div(Add(abs_x, abs_x), Sub(kOne, abs_x))),
Xor(kHalf, sign));
}
template <class D, class V>
inline __attribute__((always_inline)) V Cos(const D d, V x) {
using T = TFromD<D>;
impl::CosSinImpl<T> impl;
const V kOneOverPi = Set(d, static_cast<T>(0.31830988618379067153));
const Rebind<int32_t, D> di32;
using VI32 = decltype(Zero(di32));
const VI32 kOne = Set(di32, 1);
const V y = Abs(x);
const VI32 q = Add(ShiftLeft<1>(impl.ToInt32(d, Mul(y, kOneOverPi))), kOne);
return impl.Poly(
d, Xor(impl.CosReduce(d, y, q), impl.CosSignFromQuadrant(d, q)));
}
template <class D, class V>
inline __attribute__((always_inline)) V Exp(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kLowerBound =
Set(d, static_cast<T>((sizeof(T) == 4 ? -104.0 : -1000.0)));
const V kNegZero = Set(d, static_cast<T>(-0.0));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kOneOverLog2 = Set(d, static_cast<T>(+1.442695040888963407359924681));
impl::ExpImpl<T> impl;
const auto q =
impl.ToInt32(d, MulAdd(x, kOneOverLog2, Or(kHalf, And(x, kNegZero))));
const V y = impl.LoadExpShortRange(
d, Add(impl.ExpPoly(d, impl.ExpReduce(d, x, q)), kOne), q);
return IfThenElseZero(Ge(x, kLowerBound), y);
}
template <class D, class V>
inline __attribute__((always_inline)) V Expm1(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kLowerBound =
Set(d, static_cast<T>((sizeof(T) == 4 ? -104.0 : -1000.0)));
const V kLn2Over2 = Set(d, static_cast<T>(+0.346573590279972654708616));
const V kNegOne = Set(d, static_cast<T>(-1.0));
const V kNegZero = Set(d, static_cast<T>(-0.0));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kOneOverLog2 = Set(d, static_cast<T>(+1.442695040888963407359924681));
impl::ExpImpl<T> impl;
const auto q =
impl.ToInt32(d, MulAdd(x, kOneOverLog2, Or(kHalf, And(x, kNegZero))));
const V y = impl.ExpPoly(d, impl.ExpReduce(d, x, q));
const V z = IfThenElse(Lt(Abs(x), kLn2Over2), y,
Sub(impl.LoadExpShortRange(d, Add(y, kOne), q), kOne));
return IfThenElse(Lt(x, kLowerBound), kNegOne, z);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log(const D d, V x) {
return impl::Log<D, V, true>(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log10(const D d, V x) {
using T = TFromD<D>;
return Mul(Log(d, x), Set(d, static_cast<T>(0.4342944819032518276511)));
}
template <class D, class V>
inline __attribute__((always_inline)) V Log1p(const D d, V x) {
using T = TFromD<D>;
const V kOne = Set(d, static_cast<T>(+1.0));
const V y = Add(x, kOne);
const auto is_pole = Eq(y, kOne);
const auto divisor = Sub(IfThenZeroElse(is_pole, y), kOne);
const auto non_pole =
Mul(impl::Log<D, V, false>(d, y), Div(x, divisor));
return IfThenElse(is_pole, x, non_pole);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log2(const D d, V x) {
using T = TFromD<D>;
return Mul(Log(d, x), Set(d, static_cast<T>(1.44269504088896340735992)));
}
template <class D, class V>
inline __attribute__((always_inline)) V Sin(const D d, V x) {
using T = TFromD<D>;
impl::CosSinImpl<T> impl;
const V kOneOverPi = Set(d, static_cast<T>(0.31830988618379067153));
const V kHalf = Set(d, static_cast<T>(0.5));
const Rebind<int32_t, D> di32;
using VI32 = decltype(Zero(di32));
const V abs_x = Abs(x);
const V sign_x = Xor(abs_x, x);
const VI32 q = impl.ToInt32(d, MulAdd(abs_x, kOneOverPi, kHalf));
return impl.Poly(d, Xor(impl.SinReduce(d, abs_x, q),
Xor(impl.SinSignFromQuadrant(d, q), sign_x)));
}
template <class D, class V>
inline __attribute__((always_inline)) V Sinh(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kTwo = Set(d, static_cast<T>(+2.0));
const V sign = And(SignBit(d), x);
const V abs_x = Xor(x, sign);
const V y = Expm1(d, abs_x);
const V z = Mul(Div(Add(y, kTwo), Add(y, kOne)), Mul(y, kHalf));
return Xor(z, sign);
}
template <class D, class V>
inline __attribute__((always_inline)) V Tanh(const D d, V x) {
using T = TFromD<D>;
const V kLimit = Set(d, static_cast<T>(18.714973875));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kTwo = Set(d, static_cast<T>(+2.0));
const V sign = And(SignBit(d), x);
const V abs_x = Xor(x, sign);
const V y = Expm1(d, Mul(abs_x, kTwo));
const V z = IfThenElse(Gt(abs_x, kLimit), kOne, Div(y, Add(y, kTwo)));
return Xor(z, sign);
}
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC push_options
#pragma GCC target "sse2,ssse3" ",sse4.1,sse4.2" ",pclmul,aes"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSE4 {
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Print(const D d, const char* caption, V v, size_t lane_u = 0,
size_t max_lanes = 7) {
const size_t N = Lanes(d);
using T = TFromD<D>;
auto lanes = AllocateAligned<T>(N);
Store(v, d, lanes.get());
const auto info = hwy::detail::MakeTypeInfo<T>();
hwy::detail::PrintArray(info, caption, lanes.get(), N, lane_u, max_lanes);
}
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC push_options
#pragma GCC target "sse2,ssse3" ",sse4.1,sse4.2" ",pclmul,aes"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSE4 {
template <class D, typename T = TFromD<D>>
inline __attribute__((always_inline)) void AssertVecEqual(D d, const T* expected, Vec<D> actual,
const char* filename, const int line) {
const size_t N = Lanes(d);
auto actual_lanes = AllocateAligned<T>(N);
Store(actual, d, actual_lanes.get());
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName((1LL << 11));
hwy::detail::AssertArrayEqual(info, expected, actual_lanes.get(), N,
target_name, filename, line);
}
template <class D, typename T = TFromD<D>>
inline __attribute__((always_inline)) void AssertVecEqual(D d, Vec<D> expected, Vec<D> actual,
const char* filename, int line) {
const size_t N = Lanes(d);
auto expected_lanes = AllocateAligned<T>(N);
auto actual_lanes = AllocateAligned<T>(N);
Store(expected, d, expected_lanes.get());
Store(actual, d, actual_lanes.get());
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName((1LL << 11));
hwy::detail::AssertArrayEqual(info, expected_lanes.get(), actual_lanes.get(),
N, target_name, filename, line);
}
template <class D>
__attribute__((noinline)) void AssertMaskEqual(D d, VecArg<Mask<D>> a, VecArg<Mask<D>> b,
const char* filename, int line) {
const Vec<D> va = VecFromMask(d, a);
const Vec<D> vb = VecFromMask(d, b);
AssertVecEqual(d, va, vb, filename, line);
const char* target_name = hwy::TargetName((1LL << 11));
AssertEqual(CountTrue(d, a), CountTrue(d, b), target_name, filename, line);
AssertEqual(AllTrue(d, a), AllTrue(d, b), target_name, filename, line);
AssertEqual(AllFalse(d, a), AllFalse(d, b), target_name, filename, line);
const size_t N = Lanes(d);
const Repartition<uint8_t, D> d8;
const size_t N8 = Lanes(d8);
auto bits_a = AllocateAligned<uint8_t>(((size_t{8}) > (N8) ? (size_t{8}) : (N8)));
auto bits_b = AllocateAligned<uint8_t>(size_t{((8) > (N8) ? (8) : (N8))});
memset(bits_a.get(), 0, N8);
memset(bits_b.get(), 0, N8);
const size_t num_bytes_a = StoreMaskBits(d, a, bits_a.get());
const size_t num_bytes_b = StoreMaskBits(d, b, bits_b.get());
AssertEqual(num_bytes_a, num_bytes_b, target_name, filename, line);
size_t i = 0;
for (; i < N / 8; ++i) {
if (bits_a[i] != bits_b[i]) {
fprintf(
stderr
, "Mismatch in byte %d: %d != %d\n", static_cast<int>(i),
bits_a[i], bits_b[i]);
Print(d8, "expect", Load(d8, bits_a.get()), 0, N8);
Print(d8, "actual", Load(d8, bits_b.get()), 0, N8);
hwy::Abort(filename, line, "Masks not equal");
}
}
const size_t remainder = N % 8;
if (remainder != 0) {
const int mask = (1 << remainder) - 1;
const int valid_a = bits_a[i] & mask;
const int valid_b = bits_b[i] & mask;
if (valid_a != valid_b) {
fprintf(
stderr
, "Mismatch in last byte %d: %d != %d\n",
static_cast<int>(i), valid_a, valid_b);
Print(d8, "expect", Load(d8, bits_a.get()), 0, N8);
Print(d8, "actual", Load(d8, bits_b.get()), 0, N8);
hwy::Abort(filename, line, "Masks not equal");
}
}
}
template <class D>
inline __attribute__((always_inline)) Mask<D> MaskTrue(const D d) {
return FirstN(d, Lanes(d));
}
template <class D>
inline __attribute__((always_inline)) Mask<D> MaskFalse(const D d) {
const RebindToSigned<D> di;
const Vec<decltype(di)> zero = Zero(di);
return RebindMask(d, Lt(zero, zero));
}
namespace detail {
template <typename T, size_t kMul, size_t kMinArg, class Test, int kPow2 = 0>
struct ForeachCappedR {
static void Do(size_t min_lanes, size_t max_lanes) {
const CappedTag<T, kMul * kMinArg, kPow2> d;
const size_t lanes = Lanes(d);
if (lanes < min_lanes) return;
if (lanes <= max_lanes) {
Test()(T(), d);
}
ForeachCappedR<T, kMul / 2, kMinArg, Test, kPow2>::Do(min_lanes, max_lanes);
}
};
template <typename T, size_t kMinArg, class Test, int kPow2>
struct ForeachCappedR<T, 0, kMinArg, Test, kPow2> {
static void Do(size_t, size_t) {}
};
}
template <class Test>
class ForMaxPow2 {
mutable bool called_ = false;
public:
~ForMaxPow2() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 291, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
detail::ForeachCappedR<T, (16 / sizeof(T)), 1, Test>::Do(
1, Lanes(ScalableTag<T>()));
}
};
template <class Test, int kPow2 = 1>
class ForExtendableVectors {
mutable bool called_ = false;
public:
~ForExtendableVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 324, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMaxCapped = (16 / sizeof(T));
const size_t max_lanes = Lanes(ScalableTag<T>()) >> kPow2;
(void)kMaxCapped;
(void)max_lanes;
constexpr size_t kMul = kMaxCapped >> kPow2;
constexpr size_t kMinArg = size_t{1} << kPow2;
detail::ForeachCappedR<T, kMul, kMinArg, Test, -kPow2>::Do(1, max_lanes);
}
};
template <class Test, int kPow2 = 1>
class ForShrinkableVectors {
mutable bool called_ = false;
public:
~ForShrinkableVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 358, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMinLanes = size_t{1} << kPow2;
constexpr size_t kMaxCapped = (16 / sizeof(T));
constexpr size_t max_lanes = kMaxCapped;
(void)kMinLanes;
(void)max_lanes;
(void)max_lanes;
detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kMinLanes, Test>::Do(
kMinLanes, max_lanes);
}
};
template <size_t kMinBits, class Test>
class ForGEVectors {
mutable bool called_ = false;
public:
~ForGEVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 394, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMaxCapped = (16 / sizeof(T));
constexpr size_t kMinLanes = kMinBits / 8 / sizeof(T);
constexpr size_t max_lanes = kMaxCapped;
(void)max_lanes;
detail::ForeachCappedR<T, (16 / sizeof(T)) / kMinLanes, kMinLanes, Test>::Do(
kMinLanes, max_lanes);
}
};
template <class Test>
using ForGE128Vectors = ForGEVectors<128, Test>;
template <class Test, int kPow2 = 1>
class ForPromoteVectors {
mutable bool called_ = false;
public:
~ForPromoteVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 436, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kFactor = size_t{1} << kPow2;
static_assert(kFactor >= 2 && kFactor * sizeof(T) <= sizeof(uint64_t), "");
constexpr size_t kMaxCapped = (16 / sizeof(T));
const size_t max_lanes = Lanes(ScalableTag<T>()) >> kPow2;
(void)kMaxCapped;
(void)max_lanes;
using DLargestFrom = CappedTag<T, (kMaxCapped >> kPow2) * kFactor, -kPow2>;
static_assert(DLargestFrom::kPrivateLanes <= (kMaxCapped >> kPow2),
"HWY_MAX_LANES_D(DLargestFrom) must be less than or equal to "
"(kMaxCapped >> kPow2)");
detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kFactor, Test, -kPow2>::Do(
1, max_lanes);
}
};
template <class Test, int kPow2 = 1>
class ForDemoteVectors {
mutable bool called_ = false;
public:
~ForDemoteVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 475, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMinLanes = size_t{1} << kPow2;
constexpr size_t kMaxCapped = (16 / sizeof(T));
constexpr size_t max_lanes = kMaxCapped;
(void)kMinLanes;
(void)max_lanes;
(void)max_lanes;
detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kMinLanes, Test>::Do(
kMinLanes, max_lanes);
}
};
template <class Test, int kPow2 = 1>
class ForHalfVectors {
mutable bool called_ = false;
public:
~ForHalfVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 512, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMinLanes = size_t{1} << kPow2;
constexpr size_t kMaxCapped = (16 / sizeof(T));
detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kMinLanes, Test>::Do(
kMinLanes, kMaxCapped);
}
};
template <class Test>
class ForPartialVectors {
mutable bool called_ = false;
public:
~ForPartialVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 545, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T t) const {
called_ = true;
ForExtendableVectors<Test, 0>()(t);
}
};
template <class Test>
using ForPartialFixedOrFullScalableVectors = ForPartialVectors<Test>;
template <class Func>
void ForSignedTypes(const Func& func) {
func(int8_t());
func(int16_t());
func(int32_t());
func(int64_t());
}
template <class Func>
void ForUnsignedTypes(const Func& func) {
func(uint8_t());
func(uint16_t());
func(uint32_t());
func(uint64_t());
}
template <class Func>
void ForIntegerTypes(const Func& func) {
ForSignedTypes(func);
ForUnsignedTypes(func);
}
template <class Func>
void ForFloatTypes(const Func& func) {
func(float());
func(double());
}
template <class Func>
void ForAllTypes(const Func& func) {
ForIntegerTypes(func);
ForFloatTypes(func);
}
template <class Func>
void ForUI8(const Func& func) {
func(uint8_t());
func(int8_t());
}
template <class Func>
void ForUI16(const Func& func) {
func(uint16_t());
func(int16_t());
}
template <class Func>
void ForUIF16(const Func& func) {
ForUI16(func);
func(float16_t());
}
template <class Func>
void ForUI32(const Func& func) {
func(uint32_t());
func(int32_t());
}
template <class Func>
void ForUIF32(const Func& func) {
ForUI32(func);
func(float());
}
template <class Func>
void ForUI64(const Func& func) {
func(uint64_t());
func(int64_t());
}
template <class Func>
void ForUIF64(const Func& func) {
ForUI64(func);
func(double());
}
template <class Func>
void ForUI3264(const Func& func) {
ForUI32(func);
ForUI64(func);
}
template <class Func>
void ForUIF3264(const Func& func) {
ForUIF32(func);
ForUIF64(func);
}
template <class Func>
void ForU163264(const Func& func) {
func(uint16_t());
func(uint32_t());
func(uint64_t());
}
template <class Func>
void ForUI163264(const Func& func) {
ForUI16(func);
ForUI3264(func);
}
template <class Func>
void ForUIF163264(const Func& func) {
ForUIF16(func);
ForUIF3264(func);
}
constexpr size_t AdjustedReps(size_t max_reps) {
return ((max_reps) > (2) ? (max_reps) : (2));
}
constexpr size_t AdjustedLog2Reps(size_t max_pow2) {
return max_pow2;
}
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
#pragma GCC push_options
#pragma GCC target "sse2,ssse3" ",sse4.1,sse4.2" ",pclmul,aes"
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_SSE4 {
template <class Out, class In>
inline Out BitCast(const In& in) {
static_assert(sizeof(Out) == sizeof(In), "");
Out out;
CopyBytes<sizeof(out)>(&in, &out);
return out;
}
template <class T, class D>
__attribute__((noinline)) void TestMath(const char* name, T (*fx1)(T),
Vec<D> (*fxN)(D, VecArg<Vec<D>>), D d, T min, T max,
uint64_t max_error_ulp) {
using UintT = MakeUnsigned<T>;
const UintT min_bits = BitCast<UintT>(min);
const UintT max_bits = BitCast<UintT>(max);
int range_count = 1;
UintT ranges[2][2] = {{min_bits, max_bits}, {0, 0}};
if ((min < 0.0) && (max > 0.0)) {
ranges[0][0] = BitCast<UintT>(static_cast<T>(+0.0));
ranges[0][1] = max_bits;
ranges[1][0] = BitCast<UintT>(static_cast<T>(-0.0));
ranges[1][1] = min_bits;
range_count = 2;
}
uint64_t max_ulp = 0;
constexpr UintT kSamplesPerRange = static_cast<UintT>(AdjustedReps(4000));
for (int range_index = 0; range_index < range_count; ++range_index) {
const UintT start = ranges[range_index][0];
const UintT stop = ranges[range_index][1];
const UintT step = ((1) > (((stop - start) / kSamplesPerRange)) ? (1) : (((stop - start) / kSamplesPerRange)));
for (UintT value_bits = start; value_bits <= stop; value_bits += step) {
const T value = BitCast<T>(((((start) > (value_bits) ? (start) : (value_bits))) < (stop) ? (((start) > (value_bits) ? (start) : (value_bits))) : (stop)));
const T actual = GetLane(fxN(d, Set(d, value)));
const T expected = fx1(value);
const auto ulp = hwy::detail::ComputeUlpDelta(actual, expected);
max_ulp = ((max_ulp) > (ulp) ? (max_ulp) : (ulp));
if (ulp > max_error_ulp) {
fprintf(
stderr
, "KO %s: %s(%.17g) expected %.17g actual %.17g ulp %g max ulp %u\n",
hwy::TypeName(T(), Lanes(d)).c_str(), name, value, expected,
actual, static_cast<double>(ulp),
static_cast<uint32_t>(max_error_ulp));
if( value == 5.526720574244119e-20 && expected == 5.526720574244119e-20 && actual == 0) exit(42);
}
else {
fprintf(
stderr
, "OK %s: %s(%.17g) expected %.17g actual %.17g ulp %g max ulp %u\n",
hwy::TypeName(T(), Lanes(d)).c_str(), name, value, expected,
actual, static_cast<double>(ulp),
static_cast<uint32_t>(max_error_ulp));
}
}
}
fprintf(
stderr
, "%s: %s max_ulp %g\n", hwy::TypeName(T(), Lanes(d)).c_str(),
name, static_cast<double>(max_ulp));
}
const float kNearOneF = BitCast<float>(0x3F7FFFFF);
const double kNearOneD = BitCast<double>(0x3FEFFFFFFFFFFFFFULL);
struct TestLog1p { template <class T, class D> __attribute__((noinline)) void operator()(T, D d) { if (sizeof(T) == 4) { TestMath<T, D>("Log1p", std::log1p, CallLog1p, d, +0.0f, +1e37f, 3); } else { TestMath<T, D>("Log1p", std::log1p, CallLog1p, d, static_cast<T>(+0.0), static_cast<T>(+double(1.79769313486231570814527423731704357e+308L)
), 2); } } }; __attribute__((noinline)) void TestAllLog1p() { ForFloatTypes(ForPartialVectors<TestLog1p>()); }
}
}
#pragma GCC pop_options
static_assert(true, "For requiring trailing semicolon");
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_EMU128 {
template <class V>
using VecArg = V;
namespace detail {
constexpr size_t ScaleByPower(size_t N, int pow2) {
return pow2 >= 0 ? (N << pow2) : (N >> (-pow2));
}
template <typename T>
inline __attribute__((always_inline)) void MaybeUnpoison(T* __restrict__ unaligned, size_t count) {
(void)unaligned;
(void)count;
}
}
template <typename Lane, size_t N, int kPow2>
struct Simd {
constexpr Simd() = default;
using T = Lane;
private:
static_assert(sizeof(Lane) <= 8, "Lanes are up to 64-bit");
static constexpr size_t kWhole = N & 0xFFFFF;
static constexpr int kFrac = static_cast<int>(N >> 20);
static_assert(kWhole <= 8 * 65536 && kFrac <= 3, "Out of range");
static_assert(kFrac == 0 || kWhole == 1, "If frac, whole must be 1");
static_assert((kWhole & (kWhole - 1)) == 0 && kWhole != 0, "Not 2^x");
static_assert(kPow2 >= -8, "Forgot kPow2 recursion terminator?");
public:
static constexpr size_t kPrivateLanes =
((size_t{1}) > (detail::ScaleByPower(kWhole, kPow2 - kFrac)) ? (size_t{1}) : (detail::ScaleByPower(kWhole, kPow2 - kFrac)));
constexpr size_t MaxLanes() const { return kPrivateLanes; }
constexpr size_t MaxBytes() const { return kPrivateLanes * sizeof(Lane); }
constexpr size_t MaxBlocks() const { return (MaxBytes() + 15) / 16; }
constexpr int Pow2() const { return kPow2; }
template <typename NewT>
static constexpr size_t RepartitionLanes() {
return (kPrivateLanes * sizeof(T) + sizeof(NewT) - 1) / sizeof(NewT);
}
template <typename NewT>
static constexpr int RebindPow2() {
return kPow2 +
((sizeof(NewT) >= sizeof(T))
? static_cast<int>(CeilLog2(sizeof(NewT) / sizeof(T)))
: -static_cast<int>(CeilLog2(sizeof(T) / sizeof(NewT))));
}
private:
template <int kNewPow2, size_t kNewMaxLanes>
static constexpr size_t WholeN() {
return detail::ScaleByPower(kNewMaxLanes, -kNewPow2);
}
template <int kNewPow2, size_t kNewMaxLanes>
static constexpr size_t FracN() {
static_assert(65536 <= (size_t{1} << 20), "Change bit shift");
return static_cast<size_t>(
1 + (((0) > (kNewPow2 - static_cast<int>(CeilLog2(kNewMaxLanes))) ? (0) : (kNewPow2 - static_cast<int>(CeilLog2(kNewMaxLanes))))
<< 20));
}
public:
template <int kNewPow2, size_t kNewMaxLanes>
static constexpr size_t NewN() {
return WholeN<kNewPow2, kNewMaxLanes>() == 0
? FracN<kNewPow2, kNewMaxLanes>()
: WholeN<kNewPow2, kNewMaxLanes>();
}
template <typename NewT>
using Rebind =
Simd<NewT, NewN<RebindPow2<NewT>(), kPrivateLanes>(), RebindPow2<NewT>()>;
template <typename NewT>
using Repartition =
Simd<NewT, NewN<kPow2, RepartitionLanes<NewT>()>(), kPow2>;
using Half = Simd<T, N, kPow2 - 1>;
using Twice = Simd<T, N, kPow2 + 1>;
};
namespace detail {
template <typename T, size_t N, int kPow2>
constexpr bool IsFull(Simd<T, N, kPow2> ) {
return N == (16 / sizeof(T)) && kPow2 == 0;
}
template <typename T, size_t N, int kPow2>
struct ClampNAndPow2 {
using type = Simd<T, ((N) < (65536) ? (N) : (65536)), ((kPow2) < (3) ? (kPow2) : (3))>;
};
template <typename T, int kPow2>
struct ScalableTagChecker {
using type = typename ClampNAndPow2<T, (16 / sizeof(T)), kPow2>::type;
};
template <typename T, size_t kLimit, int kPow2>
struct CappedTagChecker {
static_assert(kLimit != 0, "Does not make sense to have zero lanes");
static constexpr size_t kLimitPow2 = size_t{1} << hwy::FloorLog2(kLimit);
static constexpr size_t N = ((kLimitPow2) < ((16 / sizeof(T))) ? (kLimitPow2) : ((16 / sizeof(T))));
using type = typename ClampNAndPow2<T, N, kPow2>::type;
};
template <typename T, size_t kNumLanes>
struct FixedTagChecker {
static_assert(kNumLanes != 0, "Does not make sense to have zero lanes");
static_assert(kNumLanes <= (16 / sizeof(T)), "Too many lanes");
using type = Simd<T, kNumLanes, 0>;
};
}
template <typename T, int kPow2 = 0>
using ScalableTag = typename detail::ScalableTagChecker<T, kPow2>::type;
template <typename T, size_t kLimit, int kPow2 = 0>
using CappedTag = typename detail::CappedTagChecker<T, kLimit, kPow2>::type;
template <typename T, size_t kLimit, int kPow2 = 0>
using CappedTagIfFixed = CappedTag<T, kLimit, kPow2>;
template <typename T, size_t kNumLanes>
using FixedTag = typename detail::FixedTagChecker<T, kNumLanes>::type;
template <typename T>
using Full16 = Simd<T, 2 / sizeof(T), 0>;
template <typename T>
using Full32 = Simd<T, 4 / sizeof(T), 0>;
template <typename T>
using Full64 = Simd<T, 8 / sizeof(T), 0>;
template <typename T>
using Full128 = Simd<T, 16 / sizeof(T), 0>;
template <class D>
using TFromD = typename D::T;
template <class D>
inline __attribute__((always_inline)) __attribute__((unused)) constexpr size_t MaxLanes(D) {
return D::kPrivateLanes;
}
template <class D>
inline __attribute__((always_inline)) __attribute__((unused)) constexpr size_t Lanes(D) {
return D::kPrivateLanes;
}
template <class T, class D>
using Rebind = typename D::template Rebind<T>;
template <class D>
using RebindToSigned = Rebind<MakeSigned<TFromD<D>>, D>;
template <class D>
using RebindToUnsigned = Rebind<MakeUnsigned<TFromD<D>>, D>;
template <class D>
using RebindToFloat = Rebind<MakeFloat<TFromD<D>>, D>;
template <class T, class D>
using Repartition = typename D::template Repartition<T>;
template <class D>
using RepartitionToWide = Repartition<MakeWide<TFromD<D>>, D>;
template <class D>
using RepartitionToNarrow = Repartition<MakeNarrow<TFromD<D>>, D>;
template <class D>
using Half = typename D::Half;
template <class D>
using Twice = typename D::Twice;
template <class D>
using BlockDFromD =
Simd<TFromD<D>, ((16 / sizeof(TFromD<D>)) < (D::kPrivateLanes) ? (16 / sizeof(TFromD<D>)) : (D::kPrivateLanes)), 0>;
}
}
static_assert(true, "For requiring trailing semicolon");
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_EMU128 {
template <typename T>
using Full128 = Simd<T, 16 / sizeof(T), 0>;
template <typename T, size_t N = 16 / sizeof(T)>
struct Vec128 {
using PrivateT = T;
static constexpr size_t kPrivateN = N;
inline __attribute__((always_inline)) Vec128() = default;
Vec128(const Vec128&) = default;
Vec128& operator=(const Vec128&) = default;
inline __attribute__((always_inline)) Vec128& operator*=(const Vec128 other) {
return *this = (*this * other);
}
inline __attribute__((always_inline)) Vec128& operator/=(const Vec128 other) {
return *this = (*this / other);
}
inline __attribute__((always_inline)) Vec128& operator+=(const Vec128 other) {
return *this = (*this + other);
}
inline __attribute__((always_inline)) Vec128& operator-=(const Vec128 other) {
return *this = (*this - other);
}
inline __attribute__((always_inline)) Vec128& operator&=(const Vec128 other) {
return *this = (*this & other);
}
inline __attribute__((always_inline)) Vec128& operator|=(const Vec128 other) {
return *this = (*this | other);
}
inline __attribute__((always_inline)) Vec128& operator^=(const Vec128 other) {
return *this = (*this ^ other);
}
T raw[16 / sizeof(T)] = {};
};
template <typename T, size_t N = 16 / sizeof(T)>
struct Mask128 {
using Raw = hwy::MakeUnsigned<T>;
static inline __attribute__((always_inline)) Raw FromBool(bool b) {
return b ? static_cast<Raw>(~Raw{0}) : 0;
}
Raw bits[16 / sizeof(T)] = {};
};
template <class V>
using DFromV = Simd<typename V::PrivateT, V::kPrivateN, 0>;
template <class V>
using TFromV = typename V::PrivateT;
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<TFromD<D>, D::kPrivateLanes> Zero(D ) {
Vec128<TFromD<D>, D::kPrivateLanes> v;
return v;
}
template <class D>
using VFromD = decltype(Zero(D()));
template <class D>
struct Vec2 {
VFromD<D> v0;
VFromD<D> v1;
};
template <class D>
struct Vec3 {
VFromD<D> v0;
VFromD<D> v1;
VFromD<D> v2;
};
template <class D>
struct Vec4 {
VFromD<D> v0;
VFromD<D> v1;
VFromD<D> v2;
VFromD<D> v3;
};
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec2<D> Create2(D , VFromD<D> v0, VFromD<D> v1) {
return Vec2<D>{v0, v1};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec3<D> Create3(D , VFromD<D> v0, VFromD<D> v1, VFromD<D> v2) {
return Vec3<D>{v0, v1, v2};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec4<D> Create4(D , VFromD<D> v0, VFromD<D> v1, VFromD<D> v2,
VFromD<D> v3) {
return Vec4<D>{v0, v1, v2, v3};
}
template <size_t kIndex, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Get2(Vec2<D> tuple) {
static_assert(kIndex < 2, "Tuple index out of bounds");
return kIndex == 0 ? tuple.v0 : tuple.v1;
}
template <size_t kIndex, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Get3(Vec3<D> tuple) {
static_assert(kIndex < 3, "Tuple index out of bounds");
return kIndex == 0 ? tuple.v0 : kIndex == 1 ? tuple.v1 : tuple.v2;
}
template <size_t kIndex, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Get4(Vec4<D> tuple) {
static_assert(kIndex < 4, "Tuple index out of bounds");
return kIndex == 0 ? tuple.v0
: kIndex == 1 ? tuple.v1
: kIndex == 2 ? tuple.v2
: tuple.v3;
}
template <class D, class VFrom>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> BitCast(D , VFrom v) {
VFromD<D> to;
CopySameSize(&v, &to);
return to;
}
template <class D, class VFrom>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ResizeBitCast(D d, VFrom v) {
using DFrom = DFromV<VFrom>;
using TFrom = TFromD<DFrom>;
using TTo = TFromD<D>;
constexpr size_t kFromByteLen = sizeof(TFrom) * DFrom::kPrivateLanes;
constexpr size_t kToByteLen = sizeof(TTo) * D::kPrivateLanes;
constexpr size_t kCopyByteLen = ((kFromByteLen) < (kToByteLen) ? (kFromByteLen) : (kToByteLen));
VFromD<D> to = Zero(d);
CopyBytes<kCopyByteLen>(&v, &to);
return to;
}
namespace detail {
template <class FromSizeTag, class ToSizeTag, class DTo, class DFrom>
inline __attribute__((always_inline)) VFromD<DTo> ZeroExtendResizeBitCast(FromSizeTag ,
ToSizeTag ,
DTo d_to, DFrom ,
VFromD<DFrom> v) {
return ResizeBitCast(d_to, v);
}
}
template <class D, typename T2>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Set(D d, const T2 t) {
VFromD<D> v;
for (size_t i = 0; i < MaxLanes(d); ++i) {
v.raw[i] = static_cast<TFromD<D>>(t);
}
return v;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Undefined(D d) {
return Zero(d);
}
template <class D, typename T = TFromD<D>, typename T2>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Iota(D d, T2 first) {
VFromD<D> v;
for (size_t i = 0; i < MaxLanes(d); ++i) {
v.raw[i] =
AddWithWraparound(hwy::IsFloatTag<T>(), static_cast<T>(first), i);
}
return v;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Not(Vec128<T, N> v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
VFromD<decltype(du)> vu = BitCast(du, v);
for (size_t i = 0; i < N; ++i) {
vu.raw[i] = static_cast<TU>(~vu.raw[i]);
}
return BitCast(d, vu);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> And(Vec128<T, N> a, Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const RebindToUnsigned<decltype(d)> du;
auto au = BitCast(du, a);
auto bu = BitCast(du, b);
for (size_t i = 0; i < N; ++i) {
au.raw[i] &= bu.raw[i];
}
return BitCast(d, au);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator&(Vec128<T, N> a, Vec128<T, N> b) {
return And(a, b);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> AndNot(Vec128<T, N> a, Vec128<T, N> b) {
return And(Not(a), b);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Or(Vec128<T, N> a, Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const RebindToUnsigned<decltype(d)> du;
auto au = BitCast(du, a);
auto bu = BitCast(du, b);
for (size_t i = 0; i < N; ++i) {
au.raw[i] |= bu.raw[i];
}
return BitCast(d, au);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator|(Vec128<T, N> a, Vec128<T, N> b) {
return Or(a, b);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Xor(Vec128<T, N> a, Vec128<T, N> b) {
const DFromV<decltype(a)> d;
const RebindToUnsigned<decltype(d)> du;
auto au = BitCast(du, a);
auto bu = BitCast(du, b);
for (size_t i = 0; i < N; ++i) {
au.raw[i] ^= bu.raw[i];
}
return BitCast(d, au);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator^(Vec128<T, N> a, Vec128<T, N> b) {
return Xor(a, b);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Xor3(Vec128<T, N> x1, Vec128<T, N> x2, Vec128<T, N> x3) {
return Xor(x1, Xor(x2, x3));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Or3(Vec128<T, N> o1, Vec128<T, N> o2, Vec128<T, N> o3) {
return Or(o1, Or(o2, o3));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> OrAnd(Vec128<T, N> o, Vec128<T, N> a1, Vec128<T, N> a2) {
return Or(o, And(a1, a2));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfVecThenElse(Vec128<T, N> mask, Vec128<T, N> yes,
Vec128<T, N> no) {
return Or(And(mask, yes), AndNot(mask, no));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> CopySign(Vec128<T, N> magn, Vec128<T, N> sign) {
static_assert(IsFloat<T>(), "Only makes sense for floating-point");
const DFromV<decltype(magn)> d;
return BitwiseIfThenElse(SignBit(d), sign, magn);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> CopySignToAbs(Vec128<T, N> abs, Vec128<T, N> sign) {
static_assert(IsFloat<T>(), "Only makes sense for floating-point");
const DFromV<decltype(abs)> d;
return OrAnd(abs, SignBit(d), sign);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> BroadcastSignBit(Vec128<T, N> v) {
for (size_t i = 0; i < N; ++i) {
v.raw[i] = v.raw[i] < 0 ? T(-1) : T(0);
}
return v;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> MaskFromVec(Vec128<T, N> v) {
Mask128<T, N> mask;
CopySameSize(&v, &mask);
return mask;
}
template <class D>
using MFromD = decltype(MaskFromVec(VFromD<D>()));
template <class DTo, class MFrom>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<DTo> RebindMask(DTo , MFrom mask) {
MFromD<DTo> to;
CopySameSize(&mask, &to);
return to;
}
template <typename T, size_t N>
Vec128<T, N> VecFromMask(Mask128<T, N> mask) {
Vec128<T, N> v;
CopySameSize(&mask, &v);
return v;
}
template <class D>
VFromD<D> VecFromMask(D , MFromD<D> mask) {
return VecFromMask(mask);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> FirstN(D d, size_t n) {
MFromD<D> m;
for (size_t i = 0; i < MaxLanes(d); ++i) {
m.bits[i] = MFromD<D>::FromBool(i < n);
}
return m;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfThenElse(Mask128<T, N> mask, Vec128<T, N> yes,
Vec128<T, N> no) {
return IfVecThenElse(VecFromMask(mask), yes, no);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfThenElseZero(Mask128<T, N> mask, Vec128<T, N> yes) {
const DFromV<decltype(yes)> d;
return IfVecThenElse(VecFromMask(mask), yes, Zero(d));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfThenZeroElse(Mask128<T, N> mask, Vec128<T, N> no) {
const DFromV<decltype(no)> d;
return IfVecThenElse(VecFromMask(mask), Zero(d), no);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> IfNegativeThenElse(Vec128<T, N> v, Vec128<T, N> yes,
Vec128<T, N> no) {
for (size_t i = 0; i < N; ++i) {
v.raw[i] = v.raw[i] < 0 ? yes.raw[i] : no.raw[i];
}
return v;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ZeroIfNegative(Vec128<T, N> v) {
const DFromV<decltype(v)> d;
return IfNegativeThenElse(v, Zero(d), v);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> Not(Mask128<T, N> m) {
return MaskFromVec(Not(VecFromMask(Simd<T, N, 0>(), m)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> And(Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(And(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> AndNot(Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(AndNot(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> Or(Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(Or(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> Xor(Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(Xor(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> ExclusiveNeither(Mask128<T, N> a, Mask128<T, N> b) {
const Simd<T, N, 0> d;
return MaskFromVec(AndNot(VecFromMask(d, a), Not(VecFromMask(d, b))));
}
template <int kBits, typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftLeft(Vec128<T, N> v) {
static_assert(0 <= kBits && kBits < sizeof(T) * 8, "Invalid shift");
for (size_t i = 0; i < N; ++i) {
const auto shifted = static_cast<hwy::MakeUnsigned<T>>(v.raw[i]) << kBits;
v.raw[i] = static_cast<T>(shifted);
}
return v;
}
template <int kBits, typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftRight(Vec128<T, N> v) {
static_assert(0 <= kBits && kBits < sizeof(T) * 8, "Invalid shift");
if (IsSigned<T>()) {
using TU = hwy::MakeUnsigned<T>;
for (size_t i = 0; i < N; ++i) {
const TU shifted = static_cast<TU>(static_cast<TU>(v.raw[i]) >> kBits);
const TU sign = v.raw[i] < 0 ? static_cast<TU>(~TU{0}) : 0;
const size_t sign_shift =
static_cast<size_t>(static_cast<int>(sizeof(TU)) * 8 - 1 - kBits);
const TU upper = static_cast<TU>(sign << sign_shift);
v.raw[i] = static_cast<T>(shifted | upper);
}
} else {
for (size_t i = 0; i < N; ++i) {
v.raw[i] = static_cast<T>(v.raw[i] >> kBits);
}
}
return v;
}
template <int kBits, typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> RotateRight(const Vec128<T, N> v) {
constexpr size_t kSizeInBits = sizeof(T) * 8;
static_assert(0 <= kBits && kBits < kSizeInBits, "Invalid shift count");
if (kBits == 0) return v;
return Or(ShiftRight<kBits>(v),
ShiftLeft<((kSizeInBits - 1) < (kSizeInBits - kBits) ? (kSizeInBits - 1) : (kSizeInBits - kBits))>(v));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftLeftSame(Vec128<T, N> v, int bits) {
for (size_t i = 0; i < N; ++i) {
const auto shifted = static_cast<hwy::MakeUnsigned<T>>(v.raw[i]) << bits;
v.raw[i] = static_cast<T>(shifted);
}
return v;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftRightSame(Vec128<T, N> v, int bits) {
if (IsSigned<T>()) {
using TU = hwy::MakeUnsigned<T>;
for (size_t i = 0; i < N; ++i) {
const TU shifted = static_cast<TU>(static_cast<TU>(v.raw[i]) >> bits);
const TU sign = v.raw[i] < 0 ? static_cast<TU>(~TU{0}) : 0;
const size_t sign_shift =
static_cast<size_t>(static_cast<int>(sizeof(TU)) * 8 - 1 - bits);
const TU upper = static_cast<TU>(sign << sign_shift);
v.raw[i] = static_cast<T>(shifted | upper);
}
} else {
for (size_t i = 0; i < N; ++i) {
v.raw[i] = static_cast<T>(v.raw[i] >> bits);
}
}
return v;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator<<(Vec128<T, N> v, Vec128<T, N> bits) {
for (size_t i = 0; i < N; ++i) {
const auto shifted = static_cast<hwy::MakeUnsigned<T>>(v.raw[i])
<< bits.raw[i];
v.raw[i] = static_cast<T>(shifted);
}
return v;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator>>(Vec128<T, N> v, Vec128<T, N> bits) {
if (IsSigned<T>()) {
using TU = hwy::MakeUnsigned<T>;
for (size_t i = 0; i < N; ++i) {
const TU shifted =
static_cast<TU>(static_cast<TU>(v.raw[i]) >> bits.raw[i]);
const TU sign = v.raw[i] < 0 ? static_cast<TU>(~TU{0}) : 0;
const size_t sign_shift = static_cast<size_t>(
static_cast<int>(sizeof(TU)) * 8 - 1 - bits.raw[i]);
const TU upper = static_cast<TU>(sign << sign_shift);
v.raw[i] = static_cast<T>(shifted | upper);
}
} else {
for (size_t i = 0; i < N; ++i) {
v.raw[i] = static_cast<T>(v.raw[i] >> bits.raw[i]);
}
}
return v;
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Add(hwy::NonFloatTag , Vec128<T, N> a,
Vec128<T, N> b) {
for (size_t i = 0; i < N; ++i) {
const uint64_t a64 = static_cast<uint64_t>(a.raw[i]);
const uint64_t b64 = static_cast<uint64_t>(b.raw[i]);
a.raw[i] = static_cast<T>((a64 + b64) & static_cast<uint64_t>(~T(0)));
}
return a;
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Sub(hwy::NonFloatTag , Vec128<T, N> a,
Vec128<T, N> b) {
for (size_t i = 0; i < N; ++i) {
const uint64_t a64 = static_cast<uint64_t>(a.raw[i]);
const uint64_t b64 = static_cast<uint64_t>(b.raw[i]);
a.raw[i] = static_cast<T>((a64 - b64) & static_cast<uint64_t>(~T(0)));
}
return a;
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Add(hwy::FloatTag , Vec128<T, N> a,
Vec128<T, N> b) {
for (size_t i = 0; i < N; ++i) {
a.raw[i] += b.raw[i];
}
return a;
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Sub(hwy::FloatTag , Vec128<T, N> a,
Vec128<T, N> b) {
for (size_t i = 0; i < N; ++i) {
a.raw[i] -= b.raw[i];
}
return a;
}
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator-(Vec128<T, N> a, Vec128<T, N> b) {
return detail::Sub(hwy::IsFloatTag<T>(), a, b);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator+(Vec128<T, N> a, Vec128<T, N> b) {
return detail::Add(hwy::IsFloatTag<T>(), a, b);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, (N + 7) / 8> SumsOf8(Vec128<uint8_t, N> v) {
Vec128<uint64_t, (N + 7) / 8> sums;
for (size_t i = 0; i < N; ++i) {
sums.raw[i / 8] += v.raw[i];
}
return sums;
}
template <typename T, size_t N, hwy::EnableIf<((size_t{1} << sizeof(T)) & ((1 << 1) | (1 << 2))) != 0>* = nullptr,
hwy::EnableIf<!hwy::IsFloat<T>() && !hwy::IsSpecialFloat<T>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> SaturatedAdd(Vec128<T, N> a, Vec128<T, N> b) {
using TW = MakeSigned<MakeWide<T>>;
for (size_t i = 0; i < N; ++i) {
a.raw[i] = static_cast<T>(((((hwy::LowestValue<T>()) > (static_cast<TW>(a.raw[i]) + b.raw[i]) ? (hwy::LowestValue<T>()) : (static_cast<TW>(a.raw[i]) + b.raw[i]))) < (hwy::HighestValue<T>()) ? (((hwy::LowestValue<T>()) > (static_cast<TW>(a.raw[i]) + b.raw[i]) ? (hwy::LowestValue<T>()) : (static_cast<TW>(a.raw[i]) + b.raw[i]))) : (hwy::HighestValue<T>()))
);
}
return a;
}
template <typename T, size_t N, hwy::EnableIf<((size_t{1} << sizeof(T)) & ((1 << 1) | (1 << 2))) != 0>* = nullptr,
hwy::EnableIf<!hwy::IsFloat<T>() && !hwy::IsSpecialFloat<T>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> SaturatedSub(Vec128<T, N> a, Vec128<T, N> b) {
using TW = MakeSigned<MakeWide<T>>;
for (size_t i = 0; i < N; ++i) {
a.raw[i] = static_cast<T>(((((hwy::LowestValue<T>()) > (static_cast<TW>(a.raw[i]) - b.raw[i]) ? (hwy::LowestValue<T>()) : (static_cast<TW>(a.raw[i]) - b.raw[i]))) < (hwy::HighestValue<T>()) ? (((hwy::LowestValue<T>()) > (static_cast<TW>(a.raw[i]) - b.raw[i]) ? (hwy::LowestValue<T>()) : (static_cast<TW>(a.raw[i]) - b.raw[i]))) : (hwy::HighestValue<T>()))
);
}
return a;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> AverageRound(Vec128<T, N> a, Vec128<T, N> b) {
static_assert(!IsSigned<T>(), "Only for unsigned");
for (size_t i = 0; i < N; ++i) {
a.raw[i] = static_cast<T>((a.raw[i] + b.raw[i] + 1) / 2);
}
return a;
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Abs(SignedTag , Vec128<T, N> a) {
for (size_t i = 0; i < N; ++i) {
const T s = a.raw[i];
const T min = hwy::LimitsMin<T>();
a.raw[i] = static_cast<T>((s >= 0 || s == min) ? a.raw[i] : -s);
}
return a;
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Abs(hwy::FloatTag , Vec128<T, N> v) {
for (size_t i = 0; i < N; ++i) {
v.raw[i] = std::abs(v.raw[i]);
}
return v;
}
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Abs(Vec128<T, N> a) {
return detail::Abs(hwy::TypeTag<T>(), a);
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Min(hwy::NonFloatTag , Vec128<T, N> a,
Vec128<T, N> b) {
for (size_t i = 0; i < N; ++i) {
a.raw[i] = ((a.raw[i]) < (b.raw[i]) ? (a.raw[i]) : (b.raw[i]));
}
return a;
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Max(hwy::NonFloatTag , Vec128<T, N> a,
Vec128<T, N> b) {
for (size_t i = 0; i < N; ++i) {
a.raw[i] = ((a.raw[i]) > (b.raw[i]) ? (a.raw[i]) : (b.raw[i]));
}
return a;
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Min(hwy::FloatTag , Vec128<T, N> a,
Vec128<T, N> b) {
for (size_t i = 0; i < N; ++i) {
if (std::isnan(a.raw[i])) {
a.raw[i] = b.raw[i];
} else if (std::isnan(b.raw[i])) {
} else {
a.raw[i] = ((a.raw[i]) < (b.raw[i]) ? (a.raw[i]) : (b.raw[i]));
}
}
return a;
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Max(hwy::FloatTag , Vec128<T, N> a,
Vec128<T, N> b) {
for (size_t i = 0; i < N; ++i) {
if (std::isnan(a.raw[i])) {
a.raw[i] = b.raw[i];
} else if (std::isnan(b.raw[i])) {
} else {
a.raw[i] = ((a.raw[i]) > (b.raw[i]) ? (a.raw[i]) : (b.raw[i]));
}
}
return a;
}
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Min(Vec128<T, N> a, Vec128<T, N> b) {
return detail::Min(hwy::IsFloatTag<T>(), a, b);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Max(Vec128<T, N> a, Vec128<T, N> b) {
return detail::Max(hwy::IsFloatTag<T>(), a, b);
}
namespace detail {
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Neg(hwy::NonFloatTag , Vec128<T, N> v) {
const DFromV<decltype(v)> d;
return Zero(d) - v;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Neg(hwy::FloatTag , Vec128<T, N> v) {
const DFromV<decltype(v)> d;
return Xor(v, SignBit(d));
}
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Neg(Vec128<T, N> v) {
return detail::Neg(hwy::IsFloatTag<T>(), v);
}
namespace detail {
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Mul(hwy::FloatTag , Vec128<T, N> a,
Vec128<T, N> b) {
for (size_t i = 0; i < N; ++i) {
a.raw[i] *= b.raw[i];
}
return a;
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Mul(SignedTag , Vec128<T, N> a, Vec128<T, N> b) {
for (size_t i = 0; i < N; ++i) {
a.raw[i] = static_cast<T>(static_cast<uint64_t>(a.raw[i]) *
static_cast<uint64_t>(b.raw[i]));
}
return a;
}
template <typename T, size_t N>
inline __attribute__((always_inline)) Vec128<T, N> Mul(UnsignedTag , Vec128<T, N> a,
Vec128<T, N> b) {
for (size_t i = 0; i < N; ++i) {
a.raw[i] = static_cast<T>(static_cast<uint64_t>(a.raw[i]) *
static_cast<uint64_t>(b.raw[i]));
}
return a;
}
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator*(Vec128<T, N> a, Vec128<T, N> b) {
return detail::Mul(hwy::TypeTag<T>(), a, b);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> operator/(Vec128<T, N> a, Vec128<T, N> b) {
for (size_t i = 0; i < N; ++i) {
a.raw[i] = (b.raw[i] == T{0}) ? 0 : a.raw[i] / b.raw[i];
}
return a;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> MulHigh(Vec128<int16_t, N> a, Vec128<int16_t, N> b) {
for (size_t i = 0; i < N; ++i) {
a.raw[i] = static_cast<int16_t>((int32_t{a.raw[i]} * b.raw[i]) >> 16);
}
return a;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint16_t, N> MulHigh(Vec128<uint16_t, N> a,
Vec128<uint16_t, N> b) {
for (size_t i = 0; i < N; ++i) {
a.raw[i] = static_cast<uint16_t>(
(static_cast<uint32_t>(a.raw[i]) * static_cast<uint32_t>(b.raw[i])) >>
16);
}
return a;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int16_t, N> MulFixedPoint15(Vec128<int16_t, N> a,
Vec128<int16_t, N> b) {
for (size_t i = 0; i < N; ++i) {
a.raw[i] = static_cast<int16_t>((2 * a.raw[i] * b.raw[i] + 32768) >> 16);
}
return a;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, (N + 1) / 2> MulEven(Vec128<int32_t, N> a,
Vec128<int32_t, N> b) {
Vec128<int64_t, (N + 1) / 2> mul;
for (size_t i = 0; i < N; i += 2) {
const int64_t a64 = a.raw[i];
mul.raw[i / 2] = a64 * b.raw[i];
}
return mul;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, (N + 1) / 2> MulEven(Vec128<uint32_t, N> a,
Vec128<uint32_t, N> b) {
Vec128<uint64_t, (N + 1) / 2> mul;
for (size_t i = 0; i < N; i += 2) {
const uint64_t a64 = a.raw[i];
mul.raw[i / 2] = a64 * b.raw[i];
}
return mul;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int64_t, (N + 1) / 2> MulOdd(Vec128<int32_t, N> a,
Vec128<int32_t, N> b) {
Vec128<int64_t, (N + 1) / 2> mul;
for (size_t i = 0; i < N; i += 2) {
const int64_t a64 = a.raw[i + 1];
mul.raw[i / 2] = a64 * b.raw[i + 1];
}
return mul;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t, (N + 1) / 2> MulOdd(Vec128<uint32_t, N> a,
Vec128<uint32_t, N> b) {
Vec128<uint64_t, (N + 1) / 2> mul;
for (size_t i = 0; i < N; i += 2) {
const uint64_t a64 = a.raw[i + 1];
mul.raw[i / 2] = a64 * b.raw[i + 1];
}
return mul;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> ApproximateReciprocal(Vec128<float, N> v) {
for (size_t i = 0; i < N; ++i) {
v.raw[i] = (std::abs(v.raw[i]) == 0.0f) ? 0.0f : 1.0f / v.raw[i];
}
return v;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> AbsDiff(Vec128<float, N> a, Vec128<float, N> b) {
return Abs(a - b);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> MulAdd(Vec128<T, N> mul, Vec128<T, N> x,
Vec128<T, N> add) {
return mul * x + add;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> NegMulAdd(Vec128<T, N> mul, Vec128<T, N> x,
Vec128<T, N> add) {
return add - mul * x;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> MulSub(Vec128<T, N> mul, Vec128<T, N> x,
Vec128<T, N> sub) {
return mul * x - sub;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> NegMulSub(Vec128<T, N> mul, Vec128<T, N> x,
Vec128<T, N> sub) {
return Neg(mul) * x - sub;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<float, N> ApproximateReciprocalSqrt(Vec128<float, N> v) {
for (size_t i = 0; i < N; ++i) {
const float half = v.raw[i] * 0.5f;
uint32_t bits;
CopySameSize(&v.raw[i], &bits);
bits = 0x5F3759DF - (bits >> 1);
CopySameSize(&bits, &v.raw[i]);
v.raw[i] = v.raw[i] * (1.5f - (half * v.raw[i] * v.raw[i]));
}
return v;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Sqrt(Vec128<T, N> v) {
for (size_t i = 0; i < N; ++i) {
v.raw[i] = std::sqrt(v.raw[i]);
}
return v;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Round(Vec128<T, N> v) {
using TI = MakeSigned<T>;
const Vec128<T, N> a = Abs(v);
for (size_t i = 0; i < N; ++i) {
if (!(a.raw[i] < MantissaEnd<T>())) {
continue;
}
const T bias = v.raw[i] < T(0.0) ? T(-0.5) : T(0.5);
const TI rounded = static_cast<TI>(v.raw[i] + bias);
if (rounded == 0) {
v.raw[i] = v.raw[i] < 0 ? T{-0} : T{0};
continue;
}
const T rounded_f = static_cast<T>(rounded);
if ((rounded & 1) && std::abs(rounded_f - v.raw[i]) == T(0.5)) {
v.raw[i] = static_cast<T>(rounded - (v.raw[i] < T(0) ? -1 : 1));
continue;
}
v.raw[i] = rounded_f;
}
return v;
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<int32_t, N> NearestInt(Vec128<float, N> v) {
using T = float;
using TI = int32_t;
const Vec128<float, N> abs = Abs(v);
Vec128<int32_t, N> ret;
for (size_t i = 0; i < N; ++i) {
const bool signbit = std::signbit(v.raw[i]);
if (!(abs.raw[i] < MantissaEnd<T>())) {
if (!(abs.raw[i] <= static_cast<T>(LimitsMax<TI>()))) {
ret.raw[i] = signbit ? LimitsMin<TI>() : LimitsMax<TI>();
continue;
}
ret.raw[i] = static_cast<TI>(v.raw[i]);
continue;
}
const T bias = v.raw[i] < T(0.0) ? T(-0.5) : T(0.5);
const TI rounded = static_cast<TI>(v.raw[i] + bias);
if (rounded == 0) {
ret.raw[i] = 0;
continue;
}
const T rounded_f = static_cast<T>(rounded);
if ((rounded & 1) && std::abs(rounded_f - v.raw[i]) == T(0.5)) {
ret.raw[i] = rounded - (signbit ? -1 : 1);
continue;
}
ret.raw[i] = rounded;
}
return ret;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Trunc(Vec128<T, N> v) {
using TI = MakeSigned<T>;
const Vec128<T, N> abs = Abs(v);
for (size_t i = 0; i < N; ++i) {
if (!(abs.raw[i] <= MantissaEnd<T>())) {
continue;
}
const TI truncated = static_cast<TI>(v.raw[i]);
if (truncated == 0) {
v.raw[i] = v.raw[i] < 0 ? -T{0} : T{0};
continue;
}
v.raw[i] = static_cast<T>(truncated);
}
return v;
}
template <typename Float, size_t N>
Vec128<Float, N> Ceil(Vec128<Float, N> v) {
constexpr int kMantissaBits = MantissaBits<Float>();
using Bits = MakeUnsigned<Float>;
const Bits kExponentMask = MaxExponentField<Float>();
const Bits kMantissaMask = MantissaMask<Float>();
const Bits kBias = kExponentMask / 2;
for (size_t i = 0; i < N; ++i) {
const bool positive = v.raw[i] > Float(0.0);
Bits bits;
CopySameSize(&v.raw[i], &bits);
const int exponent =
static_cast<int>(((bits >> kMantissaBits) & kExponentMask) - kBias);
if (exponent >= kMantissaBits) continue;
if (exponent < 0) {
v.raw[i] = positive ? Float{1} : Float{-0.0};
continue;
}
const Bits mantissa_mask = kMantissaMask >> exponent;
if ((bits & mantissa_mask) == 0) continue;
if (positive) bits += (kMantissaMask + 1) >> exponent;
bits &= ~mantissa_mask;
CopySameSize(&bits, &v.raw[i]);
}
return v;
}
template <typename Float, size_t N>
Vec128<Float, N> Floor(Vec128<Float, N> v) {
constexpr int kMantissaBits = MantissaBits<Float>();
using Bits = MakeUnsigned<Float>;
const Bits kExponentMask = MaxExponentField<Float>();
const Bits kMantissaMask = MantissaMask<Float>();
const Bits kBias = kExponentMask / 2;
for (size_t i = 0; i < N; ++i) {
const bool negative = v.raw[i] < Float(0.0);
Bits bits;
CopySameSize(&v.raw[i], &bits);
const int exponent =
static_cast<int>(((bits >> kMantissaBits) & kExponentMask) - kBias);
if (exponent >= kMantissaBits) continue;
if (exponent < 0) {
v.raw[i] = negative ? Float(-1.0) : Float(0.0);
continue;
}
const Bits mantissa_mask = kMantissaMask >> exponent;
if ((bits & mantissa_mask) == 0) continue;
if (negative) bits += (kMantissaMask + 1) >> exponent;
bits &= ~mantissa_mask;
CopySameSize(&bits, &v.raw[i]);
}
return v;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> IsNaN(Vec128<T, N> v) {
Mask128<T, N> ret;
for (size_t i = 0; i < N; ++i) {
MakeUnsigned<T> bits;
CopySameSize(&v.raw[i], &bits);
bits += bits;
bits >>= 1;
ret.bits[i] = Mask128<T, N>::FromBool(bits > ExponentMask<T>());
}
return ret;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> IsInf(Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> d;
const RebindToSigned<decltype(d)> di;
const VFromD<decltype(di)> vi = BitCast(di, v);
return RebindMask(d, Eq(Add(vi, vi), Set(di, hwy::MaxExponentTimes2<T>())));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> IsFinite(Vec128<T, N> v) {
static_assert(IsFloat<T>(), "Only for float");
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const RebindToSigned<decltype(d)> di;
using VI = VFromD<decltype(di)>;
using VU = VFromD<decltype(du)>;
const VU vu = BitCast(du, v);
const VI exp =
BitCast(di, ShiftRight<hwy::MantissaBits<T>() + 1>(Add(vu, vu)));
return RebindMask(d, Lt(exp, Set(di, hwy::MaxExponentField<T>())));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> operator==(Vec128<T, N> a, Vec128<T, N> b) {
Mask128<T, N> m;
for (size_t i = 0; i < N; ++i) {
m.bits[i] = Mask128<T, N>::FromBool(a.raw[i] == b.raw[i]);
}
return m;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> operator!=(Vec128<T, N> a, Vec128<T, N> b) {
Mask128<T, N> m;
for (size_t i = 0; i < N; ++i) {
m.bits[i] = Mask128<T, N>::FromBool(a.raw[i] != b.raw[i]);
}
return m;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> TestBit(Vec128<T, N> v, Vec128<T, N> bit) {
static_assert(!hwy::IsFloat<T>(), "Only integer vectors supported");
return (v & bit) == bit;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> operator<(Vec128<T, N> a, Vec128<T, N> b) {
Mask128<T, N> m;
for (size_t i = 0; i < N; ++i) {
m.bits[i] = Mask128<T, N>::FromBool(a.raw[i] < b.raw[i]);
}
return m;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> operator>(Vec128<T, N> a, Vec128<T, N> b) {
Mask128<T, N> m;
for (size_t i = 0; i < N; ++i) {
m.bits[i] = Mask128<T, N>::FromBool(a.raw[i] > b.raw[i]);
}
return m;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> operator<=(Vec128<T, N> a, Vec128<T, N> b) {
Mask128<T, N> m;
for (size_t i = 0; i < N; ++i) {
m.bits[i] = Mask128<T, N>::FromBool(a.raw[i] <= b.raw[i]);
}
return m;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> operator>=(Vec128<T, N> a, Vec128<T, N> b) {
Mask128<T, N> m;
for (size_t i = 0; i < N; ++i) {
m.bits[i] = Mask128<T, N>::FromBool(a.raw[i] >= b.raw[i]);
}
return m;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Lt128(D , Vec128<uint64_t> a, Vec128<uint64_t> b) {
const bool lt =
(a.raw[1] < b.raw[1]) || (a.raw[1] == b.raw[1] && a.raw[0] < b.raw[0]);
Mask128<uint64_t> ret;
ret.bits[0] = ret.bits[1] = Mask128<uint64_t>::FromBool(lt);
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Lt128Upper(D , Vec128<uint64_t> a,
Vec128<uint64_t> b) {
const bool lt = a.raw[1] < b.raw[1];
Mask128<uint64_t> ret;
ret.bits[0] = ret.bits[1] = Mask128<uint64_t>::FromBool(lt);
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Eq128(D , Vec128<uint64_t> a, Vec128<uint64_t> b) {
const bool eq = a.raw[1] == b.raw[1] && a.raw[0] == b.raw[0];
Mask128<uint64_t> ret;
ret.bits[0] = ret.bits[1] = Mask128<uint64_t>::FromBool(eq);
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<uint64_t> Ne128(D , Vec128<uint64_t> a,
Vec128<uint64_t> b) {
const bool ne = a.raw[1] != b.raw[1] || a.raw[0] != b.raw[0];
Mask128<uint64_t> ret;
ret.bits[0] = ret.bits[1] = Mask128<uint64_t>::FromBool(ne);
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Eq128Upper(D , Vec128<uint64_t> a,
Vec128<uint64_t> b) {
const bool eq = a.raw[1] == b.raw[1];
Mask128<uint64_t> ret;
ret.bits[0] = ret.bits[1] = Mask128<uint64_t>::FromBool(eq);
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> Ne128Upper(D , Vec128<uint64_t> a,
Vec128<uint64_t> b) {
const bool ne = a.raw[1] != b.raw[1];
Mask128<uint64_t> ret;
ret.bits[0] = ret.bits[1] = Mask128<uint64_t>::FromBool(ne);
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Min128(D d, VFromD<D> a, VFromD<D> b) {
return IfThenElse(Lt128(d, a, b), a, b);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Max128(D d, VFromD<D> a, VFromD<D> b) {
return IfThenElse(Lt128(d, b, a), a, b);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Min128Upper(D d, VFromD<D> a, VFromD<D> b) {
return IfThenElse(Lt128Upper(d, a, b), a, b);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Max128Upper(D d, VFromD<D> a, VFromD<D> b) {
return IfThenElse(Lt128Upper(d, b, a), a, b);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Load(D d, const TFromD<D>* __restrict__ aligned) {
VFromD<D> v;
CopyBytes<d.MaxBytes()>(aligned, v.raw);
return v;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> MaskedLoad(MFromD<D> m, D d,
const TFromD<D>* __restrict__ p) {
return IfThenElseZero(m, LoadU(d, p));
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> MaskedLoadOr(VFromD<D> v, MFromD<D> m, D d,
const TFromD<D>* __restrict__ p) {
return IfThenElse(m, LoadU(d, p), v);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> LoadU(D d, const TFromD<D>* __restrict__ p) {
return Load(d, p);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> LoadDup128(D d, const TFromD<D>* __restrict__ aligned) {
return Load(d, aligned);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Store(VFromD<D> v, D d, TFromD<D>* __restrict__ aligned) {
CopyBytes<d.MaxBytes()>(v.raw, aligned);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreU(VFromD<D> v, D d, TFromD<D>* __restrict__ p) {
Store(v, d, p);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void BlendedStore(VFromD<D> v, MFromD<D> m, D d,
TFromD<D>* __restrict__ p) {
for (size_t i = 0; i < MaxLanes(d); ++i) {
if (m.bits[i]) p[i] = v.raw[i];
}
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved2(D d, const T* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1) {
alignas(16) T buf0[MaxLanes(d)];
alignas(16) T buf1[MaxLanes(d)];
for (size_t i = 0; i < MaxLanes(d); ++i) {
buf0[i] = *unaligned++;
buf1[i] = *unaligned++;
}
v0 = Load(d, buf0);
v1 = Load(d, buf1);
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved3(D d, const T* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2) {
alignas(16) T buf0[MaxLanes(d)];
alignas(16) T buf1[MaxLanes(d)];
alignas(16) T buf2[MaxLanes(d)];
for (size_t i = 0; i < MaxLanes(d); ++i) {
buf0[i] = *unaligned++;
buf1[i] = *unaligned++;
buf2[i] = *unaligned++;
}
v0 = Load(d, buf0);
v1 = Load(d, buf1);
v2 = Load(d, buf2);
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void LoadInterleaved4(D d, const T* __restrict__ unaligned,
VFromD<D>& v0, VFromD<D>& v1, VFromD<D>& v2,
VFromD<D>& v3) {
alignas(16) T buf0[MaxLanes(d)];
alignas(16) T buf1[MaxLanes(d)];
alignas(16) T buf2[MaxLanes(d)];
alignas(16) T buf3[MaxLanes(d)];
for (size_t i = 0; i < MaxLanes(d); ++i) {
buf0[i] = *unaligned++;
buf1[i] = *unaligned++;
buf2[i] = *unaligned++;
buf3[i] = *unaligned++;
}
v0 = Load(d, buf0);
v1 = Load(d, buf1);
v2 = Load(d, buf2);
v3 = Load(d, buf3);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved2(VFromD<D> v0, VFromD<D> v1, D d,
TFromD<D>* __restrict__ unaligned) {
for (size_t i = 0; i < MaxLanes(d); ++i) {
*unaligned++ = v0.raw[i];
*unaligned++ = v1.raw[i];
}
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved3(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2, D d,
TFromD<D>* __restrict__ unaligned) {
for (size_t i = 0; i < MaxLanes(d); ++i) {
*unaligned++ = v0.raw[i];
*unaligned++ = v1.raw[i];
*unaligned++ = v2.raw[i];
}
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void StoreInterleaved4(VFromD<D> v0, VFromD<D> v1, VFromD<D> v2,
VFromD<D> v3, D d,
TFromD<D>* __restrict__ unaligned) {
for (size_t i = 0; i < MaxLanes(d); ++i) {
*unaligned++ = v0.raw[i];
*unaligned++ = v1.raw[i];
*unaligned++ = v2.raw[i];
*unaligned++ = v3.raw[i];
}
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Stream(VFromD<D> v, D d, TFromD<D>* __restrict__ aligned) {
Store(v, d, aligned);
}
template <class D, typename T = TFromD<D>, typename Offset>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void ScatterOffset(VFromD<D> v, D d, T* base,
Vec128<Offset, D::kPrivateLanes> offset) {
static_assert(sizeof(T) == sizeof(Offset), "Index/lane size must match");
for (size_t i = 0; i < MaxLanes(d); ++i) {
uint8_t* const base8 = reinterpret_cast<uint8_t*>(base) + offset.raw[i];
CopyBytes<sizeof(T)>(&v.raw[i], base8);
}
}
template <class D, typename T = TFromD<D>, typename Index>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void ScatterIndex(VFromD<D> v, D d, T* __restrict__ base,
Vec128<Index, D::kPrivateLanes> index) {
static_assert(sizeof(T) == sizeof(Index), "Index/lane size must match");
for (size_t i = 0; i < MaxLanes(d); ++i) {
base[index.raw[i]] = v.raw[i];
}
}
template <class D, typename T = TFromD<D>, typename Offset>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> GatherOffset(D d, const T* base,
Vec128<Offset, D::kPrivateLanes> offset) {
static_assert(sizeof(T) == sizeof(Offset), "Index/lane size must match");
VFromD<D> v;
for (size_t i = 0; i < MaxLanes(d); ++i) {
const uint8_t* base8 =
reinterpret_cast<const uint8_t*>(base) + offset.raw[i];
CopyBytes<sizeof(T)>(base8, &v.raw[i]);
}
return v;
}
template <class D, typename T = TFromD<D>, typename Index>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> GatherIndex(D d, const T* __restrict__ base,
Vec128<Index, D::kPrivateLanes> index) {
static_assert(sizeof(T) == sizeof(Index), "Index/lane size must match");
VFromD<D> v;
for (size_t i = 0; i < MaxLanes(d); ++i) {
v.raw[i] = base[index.raw[i]];
}
return v;
}
template <class DTo, typename TFrom, hwy::EnableIf<!hwy::IsSpecialFloat<TFrom>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DTo> PromoteTo(DTo d, Vec128<TFrom, DTo::kPrivateLanes> from) {
static_assert(sizeof(TFromD<DTo>) > sizeof(TFrom), "Not promoting");
VFromD<DTo> ret;
for (size_t i = 0; i < MaxLanes(d); ++i) {
ret.raw[i] = static_cast<TFromD<DTo>>(from.raw[i]);
}
return ret;
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D d, VFromD<Rebind<double, D>> from) {
VFromD<D> ret;
for (size_t i = 0; i < MaxLanes(d); ++i) {
if (std::isinf(from.raw[i]) ||
std::fabs(from.raw[i]) > static_cast<double>(HighestValue<float>())) {
ret.raw[i] = std::signbit(from.raw[i]) ? LowestValue<float>()
: HighestValue<float>();
continue;
}
ret.raw[i] = static_cast<float>(from.raw[i]);
}
return ret;
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D d, VFromD<Rebind<double, D>> from) {
VFromD<D> ret;
for (size_t i = 0; i < MaxLanes(d); ++i) {
if (std::isinf(from.raw[i]) ||
std::fabs(from.raw[i]) > static_cast<double>(HighestValue<int32_t>())) {
ret.raw[i] = std::signbit(from.raw[i]) ? LowestValue<int32_t>()
: HighestValue<int32_t>();
continue;
}
ret.raw[i] = static_cast<int32_t>(from.raw[i]);
}
return ret;
}
template <class DTo, typename TFrom, size_t N, hwy::EnableIf<IsSigned<TFrom>() && !IsFloat<TFrom>() && !IsSpecialFloat<TFrom>()>* = nullptr,
hwy::EnableIf<!hwy::IsFloat<TFromD<DTo> >() && !hwy::IsSpecialFloat<TFromD<DTo> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DTo> DemoteTo(DTo , Vec128<TFrom, N> from) {
using TTo = TFromD<DTo>;
static_assert(sizeof(TTo) < sizeof(TFrom), "Not demoting");
VFromD<DTo> ret;
for (size_t i = 0; i < N; ++i) {
from.raw[i] =
((((LimitsMin<TTo>()) > (from.raw[i]) ? (LimitsMin<TTo>()) : (from.raw[i]))) < (LimitsMax<TTo>()) ? (((LimitsMin<TTo>()) > (from.raw[i]) ? (LimitsMin<TTo>()) : (from.raw[i]))) : (LimitsMax<TTo>()));
ret.raw[i] = static_cast<TTo>(from.raw[i]);
}
return ret;
}
template <class DTo, typename TFrom, size_t N, hwy::EnableIf<!IsSigned<TFrom>()>* = nullptr,
hwy::EnableIf<!IsSigned<TFromD<DTo> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DTo> DemoteTo(DTo , Vec128<TFrom, N> from) {
using TTo = TFromD<DTo>;
static_assert(sizeof(TTo) < sizeof(TFrom), "Not demoting");
VFromD<DTo> ret;
for (size_t i = 0; i < N; ++i) {
from.raw[i] = ((from.raw[i]) < (LimitsMax<TTo>()) ? (from.raw[i]) : (LimitsMax<TTo>()));
ret.raw[i] = static_cast<TTo>(from.raw[i]);
}
return ret;
}
template <class DBF16, hwy::EnableIf<IsSame<TFromD<DBF16>, bfloat16_t>()>* = nullptr, class VF32>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DBF16> ReorderDemote2To(DBF16 dbf16, VF32 a, VF32 b) {
const Repartition<uint32_t, decltype(dbf16)> du32;
const VFromD<decltype(du32)> b_in_lower = ShiftRight<16>(BitCast(du32, b));
const VFromD<decltype(du32)> a_mask = Set(du32, 0xFFFF0000);
return BitCast(dbf16, IfVecThenElse(a_mask, BitCast(du32, a), b_in_lower));
}
template <class DN, hwy::EnableIf<!hwy::IsFloat<TFromD<DN> >() && !hwy::IsSpecialFloat<TFromD<DN> >()>* = nullptr, class V,
hwy::EnableIf<IsSigned<TFromV<V> >() && !IsFloat<TFromV<V> >() && !IsSpecialFloat<TFromV<V> >()>* = nullptr, hwy::EnableIf<sizeof(TFromV<V>) == (sizeof(TFromD<DN>) * 2)>* = nullptr,
hwy::EnableIf<(DN::kPrivateLanes == DFromV<V>::kPrivateLanes * 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DN> ReorderDemote2To(DN dn, V a, V b) {
const RepartitionToWide<decltype(dn)> dw;
const size_t NW = Lanes(dw);
using TN = TFromD<DN>;
const TN min = LimitsMin<TN>();
const TN max = LimitsMax<TN>();
VFromD<DN> ret;
for (size_t i = 0; i < NW; ++i) {
ret.raw[i] = static_cast<TN>(((((min) > (a.raw[i]) ? (min) : (a.raw[i]))) < (max) ? (((min) > (a.raw[i]) ? (min) : (a.raw[i]))) : (max)));
}
for (size_t i = 0; i < NW; ++i) {
ret.raw[NW + i] = static_cast<TN>(((((min) > (b.raw[i]) ? (min) : (b.raw[i]))) < (max) ? (((min) > (b.raw[i]) ? (min) : (b.raw[i]))) : (max)));
}
return ret;
}
template <class DN, hwy::EnableIf<!IsSigned<TFromD<DN> >()>* = nullptr, class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
hwy::EnableIf<sizeof(TFromV<V>) == (sizeof(TFromD<DN>) * 2)>* = nullptr,
hwy::EnableIf<(DN::kPrivateLanes == DFromV<V>::kPrivateLanes * 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DN> ReorderDemote2To(DN dn, V a, V b) {
const RepartitionToWide<decltype(dn)> dw;
const size_t NW = Lanes(dw);
using TN = TFromD<DN>;
const TN max = LimitsMax<TN>();
VFromD<DN> ret;
for (size_t i = 0; i < NW; ++i) {
ret.raw[i] = static_cast<TN>(((a.raw[i]) < (max) ? (a.raw[i]) : (max)));
}
for (size_t i = 0; i < NW; ++i) {
ret.raw[NW + i] = static_cast<TN>(((b.raw[i]) < (max) ? (b.raw[i]) : (max)));
}
return ret;
}
template <class DN, hwy::EnableIf<!hwy::IsFloat<TFromD<DN> >() && !hwy::IsSpecialFloat<TFromD<DN> >()>* = nullptr, class V,
hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr,
hwy::EnableIf<sizeof(TFromV<V>) == (sizeof(TFromD<DN>) * 2)>* = nullptr,
hwy::EnableIf<(DN::kPrivateLanes == DFromV<V>::kPrivateLanes * 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DN> OrderedDemote2To(DN dn, V a, V b) {
return ReorderDemote2To(dn, a, b);
}
template <class DN, hwy::EnableIf<IsSame<TFromD<DN>, bfloat16_t>()>* = nullptr, class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, float>()>* = nullptr,
hwy::EnableIf<(DN::kPrivateLanes == DFromV<V>::kPrivateLanes * 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DN> OrderedDemote2To(DN dn, V a, V b) {
const RebindToUnsigned<DFromV<decltype(a)>> du32;
const size_t NW = Lanes(du32);
VFromD<Repartition<uint16_t, DN>> ret;
const auto a_bits = BitCast(du32, a);
const auto b_bits = BitCast(du32, b);
for (size_t i = 0; i < NW; ++i) {
ret.raw[i] = static_cast<uint16_t>(a_bits.raw[i] >> 16);
}
for (size_t i = 0; i < NW; ++i) {
ret.raw[NW + i] = static_cast<uint16_t>(b_bits.raw[i] >> 16);
}
return BitCast(dn, ret);
}
namespace detail {
inline __attribute__((always_inline)) void StoreU16ToF16(const uint16_t val,
hwy::float16_t* __restrict__ to) {
CopySameSize(&val, to);
}
inline __attribute__((always_inline)) uint16_t U16FromF16(const hwy::float16_t* __restrict__ from) {
uint16_t bits16;
CopySameSize(from, &bits16);
return bits16;
}
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , Vec128<float16_t, N> v) {
VFromD<D> ret;
for (size_t i = 0; i < N; ++i) {
const uint16_t bits16 = detail::U16FromF16(&v.raw[i]);
const uint32_t sign = static_cast<uint32_t>(bits16 >> 15);
const uint32_t biased_exp = (bits16 >> 10) & 0x1F;
const uint32_t mantissa = bits16 & 0x3FF;
if (biased_exp == 0) {
const float subnormal =
(1.0f / 16384) * (static_cast<float>(mantissa) * (1.0f / 1024));
ret.raw[i] = sign ? -subnormal : subnormal;
continue;
}
const uint32_t biased_exp32 = biased_exp + (127 - 15);
const uint32_t mantissa32 = mantissa << (23 - 10);
const uint32_t bits32 = (sign << 31) | (biased_exp32 << 23) | mantissa32;
CopySameSize(&bits32, &ret.raw[i]);
}
return ret;
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> PromoteTo(D , Vec128<bfloat16_t, N> v) {
VFromD<D> ret;
for (size_t i = 0; i < N; ++i) {
ret.raw[i] = F32FromBF16(v.raw[i]);
}
return ret;
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, float16_t>()>* = nullptr, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , Vec128<float, N> v) {
VFromD<D> ret;
for (size_t i = 0; i < N; ++i) {
uint32_t bits32;
CopySameSize(&v.raw[i], &bits32);
const uint32_t sign = bits32 >> 31;
const uint32_t biased_exp32 = (bits32 >> 23) & 0xFF;
const uint32_t mantissa32 = bits32 & 0x7FFFFF;
const int32_t exp = ((static_cast<int32_t>(biased_exp32) - 127) < (15) ? (static_cast<int32_t>(biased_exp32) - 127) : (15));
if (exp < -24) {
ZeroBytes<sizeof(uint16_t)>(&ret.raw[i]);
continue;
}
uint32_t biased_exp16, mantissa16;
if (exp < -14) {
biased_exp16 = 0;
const uint32_t sub_exp = static_cast<uint32_t>(-14 - exp);
do { } while (0);
mantissa16 = static_cast<uint32_t>((1u << (10 - sub_exp)) +
(mantissa32 >> (13 + sub_exp)));
} else {
biased_exp16 = static_cast<uint32_t>(exp + 15);
do { } while (0);
mantissa16 = mantissa32 >> 13;
}
do { } while (0);
const uint32_t bits16 = (sign << 15) | (biased_exp16 << 10) | mantissa16;
do { } while (0);
const uint16_t narrowed = static_cast<uint16_t>(bits16);
detail::StoreU16ToF16(narrowed, &ret.raw[i]);
}
return ret;
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, bfloat16_t>()>* = nullptr, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> DemoteTo(D , Vec128<float, N> v) {
VFromD<D> ret;
for (size_t i = 0; i < N; ++i) {
ret.raw[i] = BF16FromF32(v.raw[i]);
}
return ret;
}
namespace detail {
template <typename TFrom, typename DTo>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DTo> ConvertTo(hwy::FloatTag , DTo ,
Vec128<TFrom, DTo::kPrivateLanes> from) {
using ToT = TFromD<DTo>;
static_assert(sizeof(ToT) == sizeof(TFrom), "Should have same size");
VFromD<DTo> ret;
constexpr size_t N = DTo::kPrivateLanes;
for (size_t i = 0; i < N; ++i) {
const double f = static_cast<double>(from.raw[i]);
if (std::isinf(from.raw[i]) ||
std::fabs(f) > static_cast<double>(LimitsMax<ToT>())) {
ret.raw[i] =
std::signbit(from.raw[i]) ? LimitsMin<ToT>() : LimitsMax<ToT>();
continue;
}
ret.raw[i] = static_cast<ToT>(from.raw[i]);
}
return ret;
}
template <typename TFrom, typename DTo>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DTo> ConvertTo(hwy::NonFloatTag , DTo ,
Vec128<TFrom, DTo::kPrivateLanes> from) {
using ToT = TFromD<DTo>;
static_assert(sizeof(ToT) == sizeof(TFrom), "Should have same size");
VFromD<DTo> ret;
constexpr size_t N = DTo::kPrivateLanes;
for (size_t i = 0; i < N; ++i) {
ret.raw[i] = static_cast<ToT>(from.raw[i]);
}
return ret;
}
}
template <class DTo, typename TFrom>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DTo> ConvertTo(DTo d, Vec128<TFrom, DTo::kPrivateLanes> from) {
return detail::ConvertTo(hwy::IsFloatTag<TFrom>(), d, from);
}
template <size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint8_t, N> U8FromU32(Vec128<uint32_t, N> v) {
return DemoteTo(Simd<uint8_t, N, 0>(), v);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TruncateTo(D , Vec128<uint64_t, N> v) {
VFromD<D> ret;
for (size_t i = 0; i < N; ++i) {
ret.raw[i] = static_cast<uint8_t>(v.raw[i] & 0xFF);
}
return ret;
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TruncateTo(D , Vec128<uint64_t, N> v) {
VFromD<D> ret;
for (size_t i = 0; i < N; ++i) {
ret.raw[i] = static_cast<uint16_t>(v.raw[i] & 0xFFFF);
}
return ret;
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TruncateTo(D , Vec128<uint64_t, N> v) {
VFromD<D> ret;
for (size_t i = 0; i < N; ++i) {
ret.raw[i] = static_cast<uint32_t>(v.raw[i] & 0xFFFFFFFFu);
}
return ret;
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TruncateTo(D , Vec128<uint32_t, N> v) {
VFromD<D> ret;
for (size_t i = 0; i < N; ++i) {
ret.raw[i] = static_cast<uint8_t>(v.raw[i] & 0xFF);
}
return ret;
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TruncateTo(D , Vec128<uint32_t, N> v) {
VFromD<D> ret;
for (size_t i = 0; i < N; ++i) {
ret.raw[i] = static_cast<uint16_t>(v.raw[i] & 0xFFFF);
}
return ret;
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TruncateTo(D , Vec128<uint16_t, N> v) {
VFromD<D> ret;
for (size_t i = 0; i < N; ++i) {
ret.raw[i] = static_cast<uint8_t>(v.raw[i] & 0xFF);
}
return ret;
}
template <class DN, hwy::EnableIf<!IsSigned<TFromD<DN> >()>* = nullptr, class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
hwy::EnableIf<sizeof(TFromV<V>) == (sizeof(TFromD<DN>) * 2)>* = nullptr,
hwy::EnableIf<(DN::kPrivateLanes == DFromV<V>::kPrivateLanes * 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DN> OrderedTruncate2To(DN dn, V a, V b) {
const RepartitionToWide<decltype(dn)> dw;
const size_t NW = Lanes(dw);
using TW = TFromD<decltype(dw)>;
using TN = TFromD<decltype(dn)>;
VFromD<DN> ret;
constexpr TW max_val{LimitsMax<TN>()};
for (size_t i = 0; i < NW; ++i) {
ret.raw[i] = static_cast<TN>(a.raw[i] & max_val);
}
for (size_t i = 0; i < NW; ++i) {
ret.raw[NW + i] = static_cast<TN>(b.raw[i] & max_val);
}
return ret;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N / 2> LowerHalf(Vec128<T, N> v) {
Vec128<T, N / 2> ret;
CopyBytes<N / 2 * sizeof(T)>(v.raw, ret.raw);
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> LowerHalf(D , VFromD<Twice<D>> v) {
return LowerHalf(v);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> UpperHalf(D d, VFromD<Twice<D>> v) {
VFromD<D> ret;
CopyBytes<d.MaxBytes()>(&v.raw[MaxLanes(d)], ret.raw);
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ZeroExtendVector(D d, VFromD<Half<D>> v) {
const Half<decltype(d)> dh;
VFromD<D> ret;
CopyBytes<dh.MaxBytes()>(v.raw, ret.raw);
return ret;
}
template <class D, class VH = VFromD<Half<D>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Combine(D d, VH hi_half, VH lo_half) {
const Half<decltype(d)> dh;
VFromD<D> ret;
CopyBytes<dh.MaxBytes()>(lo_half.raw, &ret.raw[0]);
CopyBytes<dh.MaxBytes()>(hi_half.raw, &ret.raw[MaxLanes(dh)]);
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatLowerLower(D d, VFromD<D> hi, VFromD<D> lo) {
const Half<decltype(d)> dh;
VFromD<D> ret;
CopyBytes<dh.MaxBytes()>(lo.raw, &ret.raw[0]);
CopyBytes<dh.MaxBytes()>(hi.raw, &ret.raw[MaxLanes(dh)]);
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatUpperUpper(D d, VFromD<D> hi, VFromD<D> lo) {
const Half<decltype(d)> dh;
VFromD<D> ret;
CopyBytes<dh.MaxBytes()>(&lo.raw[MaxLanes(dh)], &ret.raw[0]);
CopyBytes<dh.MaxBytes()>(&hi.raw[MaxLanes(dh)], &ret.raw[MaxLanes(dh)]);
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatLowerUpper(D d, VFromD<D> hi, VFromD<D> lo) {
const Half<decltype(d)> dh;
VFromD<D> ret;
CopyBytes<dh.MaxBytes()>(&lo.raw[MaxLanes(dh)], &ret.raw[0]);
CopyBytes<dh.MaxBytes()>(hi.raw, &ret.raw[MaxLanes(dh)]);
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatUpperLower(D d, VFromD<D> hi, VFromD<D> lo) {
const Half<decltype(d)> dh;
VFromD<D> ret;
CopyBytes<dh.MaxBytes()>(lo.raw, &ret.raw[0]);
CopyBytes<dh.MaxBytes()>(&hi.raw[MaxLanes(dh)], &ret.raw[MaxLanes(dh)]);
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatEven(D d, VFromD<D> hi, VFromD<D> lo) {
const Half<decltype(d)> dh;
VFromD<D> ret;
for (size_t i = 0; i < MaxLanes(dh); ++i) {
ret.raw[i] = lo.raw[2 * i];
}
for (size_t i = 0; i < MaxLanes(dh); ++i) {
ret.raw[MaxLanes(dh) + i] = hi.raw[2 * i];
}
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ConcatOdd(D d, VFromD<D> hi, VFromD<D> lo) {
const Half<decltype(d)> dh;
VFromD<D> ret;
for (size_t i = 0; i < MaxLanes(dh); ++i) {
ret.raw[i] = lo.raw[2 * i + 1];
}
for (size_t i = 0; i < MaxLanes(dh); ++i) {
ret.raw[MaxLanes(dh) + i] = hi.raw[2 * i + 1];
}
return ret;
}
template <int kBytes, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> CombineShiftRightBytes(D d, VFromD<D> hi, VFromD<D> lo) {
VFromD<D> ret;
const uint8_t* __restrict__ lo8 =
reinterpret_cast<const uint8_t * __restrict__>(lo.raw);
uint8_t* __restrict__ ret8 =
reinterpret_cast<uint8_t * __restrict__>(ret.raw);
CopyBytes<d.MaxBytes() - kBytes>(lo8 + kBytes, ret8);
CopyBytes<kBytes>(hi.raw, ret8 + d.MaxBytes() - kBytes);
return ret;
}
template <int kBytes, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ShiftLeftBytes(D d, VFromD<D> v) {
static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes");
VFromD<D> ret;
uint8_t* __restrict__ ret8 =
reinterpret_cast<uint8_t * __restrict__>(ret.raw);
ZeroBytes<kBytes>(ret8);
CopyBytes<d.MaxBytes() - kBytes>(v.raw, ret8 + kBytes);
return ret;
}
template <int kBytes, typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftLeftBytes(Vec128<T, N> v) {
return ShiftLeftBytes<kBytes>(DFromV<decltype(v)>(), v);
}
template <int kLanes, class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ShiftLeftLanes(D d, VFromD<D> v) {
const Repartition<uint8_t, decltype(d)> d8;
return BitCast(d, ShiftLeftBytes<kLanes * sizeof(T)>(BitCast(d8, v)));
}
template <int kLanes, typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> ShiftLeftLanes(Vec128<T, N> v) {
return ShiftLeftLanes<kLanes>(DFromV<decltype(v)>(), v);
}
template <int kBytes, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ShiftRightBytes(D d, VFromD<D> v) {
static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes");
VFromD<D> ret;
const uint8_t* __restrict__ v8 =
reinterpret_cast<const uint8_t * __restrict__>(v.raw);
uint8_t* __restrict__ ret8 =
reinterpret_cast<uint8_t * __restrict__>(ret.raw);
CopyBytes<d.MaxBytes() - kBytes>(v8 + kBytes, ret8);
ZeroBytes<kBytes>(ret8 + d.MaxBytes() - kBytes);
return ret;
}
template <int kLanes, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ShiftRightLanes(D d, VFromD<D> v) {
const Repartition<uint8_t, decltype(d)> d8;
constexpr size_t kBytes = kLanes * sizeof(TFromD<D>);
return BitCast(d, ShiftRightBytes<kBytes>(d8, BitCast(d8, v)));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T GetLane(Vec128<T, N> v) {
return v.raw[0];
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> InsertLane(Vec128<T, N> v, size_t i, T t) {
v.raw[i] = t;
return v;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T ExtractLane(Vec128<T, N> v, size_t i) {
return v.raw[i];
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> DupEven(Vec128<T, N> v) {
for (size_t i = 0; i < N; i += 2) {
v.raw[i + 1] = v.raw[i];
}
return v;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> DupOdd(Vec128<T, N> v) {
for (size_t i = 0; i < N; i += 2) {
v.raw[i] = v.raw[i + 1];
}
return v;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> OddEven(Vec128<T, N> odd, Vec128<T, N> even) {
for (size_t i = 0; i < N; i += 2) {
odd.raw[i] = even.raw[i];
}
return odd;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> OddEvenBlocks(Vec128<T, N> , Vec128<T, N> even) {
return even;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> SwapAdjacentBlocks(Vec128<T, N> v) {
return v;
}
template <typename T, size_t N>
struct Indices128 {
MakeSigned<T> raw[N];
};
template <class D, typename TI, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Indices128<TFromD<D>, N> IndicesFromVec(D d, Vec128<TI, N> vec) {
static_assert(sizeof(TFromD<D>) == sizeof(TI), "Index/lane size must match");
Indices128<TFromD<D>, N> ret;
CopyBytes<d.MaxBytes()>(vec.raw, ret.raw);
return ret;
}
template <class D, typename TI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Indices128<TFromD<D>, D::kPrivateLanes> SetTableIndices(
D d, const TI* idx) {
return IndicesFromVec(d, LoadU(Rebind<TI, D>(), idx));
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> TableLookupLanes(Vec128<T, N> v, Indices128<T, N> idx) {
Vec128<T, N> ret;
for (size_t i = 0; i < N; ++i) {
ret.raw[i] = v.raw[idx.raw[i]];
}
return ret;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> TwoTablesLookupLanes(Vec128<T, N> a, Vec128<T, N> b,
Indices128<T, N> idx) {
using TI = MakeSigned<T>;
Vec128<T, N> ret;
constexpr TI kVecLaneIdxMask = static_cast<TI>(N - 1);
for (size_t i = 0; i < N; ++i) {
const auto src_idx = idx.raw[i];
const auto masked_src_lane_idx = src_idx & kVecLaneIdxMask;
ret.raw[i] = (src_idx < static_cast<TI>(N)) ? a.raw[masked_src_lane_idx]
: b.raw[masked_src_lane_idx];
}
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReverseBlocks(D , VFromD<D> v) {
return v;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse(D d, VFromD<D> v) {
VFromD<D> ret;
for (size_t i = 0; i < MaxLanes(d); ++i) {
ret.raw[i] = v.raw[MaxLanes(d) - 1 - i];
}
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse2(D d, VFromD<D> v) {
VFromD<D> ret;
for (size_t i = 0; i < MaxLanes(d); i += 2) {
ret.raw[i + 0] = v.raw[i + 1];
ret.raw[i + 1] = v.raw[i + 0];
}
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse4(D d, VFromD<D> v) {
VFromD<D> ret;
for (size_t i = 0; i < MaxLanes(d); i += 4) {
ret.raw[i + 0] = v.raw[i + 3];
ret.raw[i + 1] = v.raw[i + 2];
ret.raw[i + 2] = v.raw[i + 1];
ret.raw[i + 3] = v.raw[i + 0];
}
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Reverse8(D d, VFromD<D> v) {
VFromD<D> ret;
for (size_t i = 0; i < MaxLanes(d); i += 8) {
ret.raw[i + 0] = v.raw[i + 7];
ret.raw[i + 1] = v.raw[i + 6];
ret.raw[i + 2] = v.raw[i + 5];
ret.raw[i + 3] = v.raw[i + 4];
ret.raw[i + 4] = v.raw[i + 3];
ret.raw[i + 5] = v.raw[i + 2];
ret.raw[i + 6] = v.raw[i + 1];
ret.raw[i + 7] = v.raw[i + 0];
}
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpLanes(D d, VFromD<D> v, size_t amt) {
VFromD<D> ret = Zero(d);
constexpr size_t N = D::kPrivateLanes;
const size_t clamped_amt = ((amt) < (N) ? (amt) : (N));
memcpy(ret.raw + clamped_amt, v.raw, (N - clamped_amt) * sizeof(TFromD<D>));
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownLanes(D d, VFromD<D> v, size_t amt) {
VFromD<D> ret = Zero(d);
constexpr size_t N = D::kPrivateLanes;
const size_t clamped_amt = ((amt) < (N) ? (amt) : (N));
memcpy(ret.raw, v.raw + clamped_amt, (N - clamped_amt) * sizeof(TFromD<D>));
return ret;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Shuffle2301(Vec128<T, N> v) {
static_assert(sizeof(T) == 4, "Only for 32-bit");
static_assert(N == 2 || N == 4, "Does not make sense for N=1");
return Reverse2(DFromV<decltype(v)>(), v);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Shuffle1032(Vec128<T> v) {
static_assert(sizeof(T) == 4, "Only for 32-bit");
Vec128<T> ret;
ret.raw[3] = v.raw[1];
ret.raw[2] = v.raw[0];
ret.raw[1] = v.raw[3];
ret.raw[0] = v.raw[2];
return ret;
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Shuffle01(Vec128<T> v) {
static_assert(sizeof(T) == 8, "Only for 64-bit");
return Reverse2(DFromV<decltype(v)>(), v);
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Shuffle0321(Vec128<T> v) {
Vec128<T> ret;
ret.raw[3] = v.raw[0];
ret.raw[2] = v.raw[3];
ret.raw[1] = v.raw[2];
ret.raw[0] = v.raw[1];
return ret;
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Shuffle2103(Vec128<T> v) {
Vec128<T> ret;
ret.raw[3] = v.raw[2];
ret.raw[2] = v.raw[1];
ret.raw[1] = v.raw[0];
ret.raw[0] = v.raw[3];
return ret;
}
template <typename T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T> Shuffle0123(Vec128<T> v) {
return Reverse4(DFromV<decltype(v)>(), v);
}
template <int kLane, typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Broadcast(Vec128<T, N> v) {
for (size_t i = 0; i < N; ++i) {
v.raw[i] = v.raw[kLane];
}
return v;
}
template <typename T, size_t N, typename TI, size_t NI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<TI, NI> TableLookupBytes(Vec128<T, N> v,
Vec128<TI, NI> indices) {
const uint8_t* __restrict__ v_bytes =
reinterpret_cast<const uint8_t * __restrict__>(v.raw);
const uint8_t* __restrict__ idx_bytes =
reinterpret_cast<const uint8_t*>(indices.raw);
Vec128<TI, NI> ret;
uint8_t* __restrict__ ret_bytes =
reinterpret_cast<uint8_t * __restrict__>(ret.raw);
for (size_t i = 0; i < NI * sizeof(TI); ++i) {
const size_t idx = idx_bytes[i];
ret_bytes[i] = idx < sizeof(T) * N ? v_bytes[idx] : 0;
}
return ret;
}
template <typename T, size_t N, typename TI, size_t NI>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<TI, NI> TableLookupBytesOr0(Vec128<T, N> v,
Vec128<TI, NI> indices) {
return TableLookupBytes(v, indices);
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> InterleaveLower(Vec128<T, N> a, Vec128<T, N> b) {
Vec128<T, N> ret;
for (size_t i = 0; i < N / 2; ++i) {
ret.raw[2 * i + 0] = a.raw[i];
ret.raw[2 * i + 1] = b.raw[i];
}
return ret;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V InterleaveLower(DFromV<V> , V a, V b) {
return InterleaveLower(a, b);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> InterleaveUpper(D d, VFromD<D> a, VFromD<D> b) {
const Half<decltype(d)> dh;
VFromD<D> ret;
for (size_t i = 0; i < MaxLanes(dh); ++i) {
ret.raw[2 * i + 0] = a.raw[MaxLanes(dh) + i];
ret.raw[2 * i + 1] = b.raw[MaxLanes(dh) + i];
}
return ret;
}
template <class V, class DW = RepartitionToWide<DFromV<V>>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DW> ZipLower(V a, V b) {
return BitCast(DW(), InterleaveLower(a, b));
}
template <class V, class D = DFromV<V>, class DW = RepartitionToWide<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DW> ZipLower(DW dw, V a, V b) {
return BitCast(dw, InterleaveLower(D(), a, b));
}
template <class V, class D = DFromV<V>, class DW = RepartitionToWide<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DW> ZipUpper(DW dw, V a, V b) {
return BitCast(dw, InterleaveUpper(D(), a, b));
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) bool AllFalse(D d, MFromD<D> mask) {
typename MFromD<D>::Raw or_sum = 0;
for (size_t i = 0; i < MaxLanes(d); ++i) {
or_sum |= mask.bits[i];
}
return or_sum == 0;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) bool AllTrue(D d, MFromD<D> mask) {
constexpr uint64_t kAll = LimitsMax<typename MFromD<D>::Raw>();
uint64_t and_sum = kAll;
for (size_t i = 0; i < MaxLanes(d); ++i) {
and_sum &= mask.bits[i];
}
return and_sum == kAll;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) MFromD<D> LoadMaskBits(D d, const uint8_t* __restrict__ bits) {
MFromD<D> m;
for (size_t i = 0; i < MaxLanes(d); ++i) {
const size_t bit = size_t{1} << (i & 7);
const size_t idx_byte = i >> 3;
m.bits[i] = MFromD<D>::FromBool((bits[idx_byte] & bit) != 0);
}
return m;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t StoreMaskBits(D d, MFromD<D> mask, uint8_t* bits) {
bits[0] = 0;
if (MaxLanes(d) > 8) bits[1] = 0;
for (size_t i = 0; i < MaxLanes(d); ++i) {
const size_t bit = size_t{1} << (i & 7);
const size_t idx_byte = i >> 3;
if (mask.bits[i]) {
bits[idx_byte] = static_cast<uint8_t>(bits[idx_byte] | bit);
}
}
return MaxLanes(d) > 8 ? 2 : 1;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CountTrue(D d, MFromD<D> mask) {
size_t count = 0;
for (size_t i = 0; i < MaxLanes(d); ++i) {
count += mask.bits[i] != 0;
}
return count;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t FindKnownFirstTrue(D d, MFromD<D> mask) {
for (size_t i = 0; i < MaxLanes(d); ++i) {
if (mask.bits[i] != 0) return i;
}
do { } while (0);
return 0;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) intptr_t FindFirstTrue(D d, MFromD<D> mask) {
for (size_t i = 0; i < MaxLanes(d); ++i) {
if (mask.bits[i] != 0) return static_cast<intptr_t>(i);
}
return intptr_t{-1};
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t FindKnownLastTrue(D d, MFromD<D> mask) {
for (intptr_t i = static_cast<intptr_t>(MaxLanes(d) - 1); i >= 0; i--) {
if (mask.bits[i] != 0) return static_cast<size_t>(i);
}
do { } while (0);
return 0;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) intptr_t FindLastTrue(D d, MFromD<D> mask) {
for (intptr_t i = static_cast<intptr_t>(MaxLanes(d) - 1); i >= 0; i--) {
if (mask.bits[i] != 0) return i;
}
return intptr_t{-1};
}
template <typename T>
struct CompressIsPartition {
enum { value = (sizeof(T) != 1) };
};
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Compress(Vec128<T, N> v, Mask128<T, N> mask) {
size_t count = 0;
Vec128<T, N> ret;
for (size_t i = 0; i < N; ++i) {
if (mask.bits[i]) {
ret.raw[count++] = v.raw[i];
}
}
for (size_t i = 0; i < N; ++i) {
if (!mask.bits[i]) {
ret.raw[count++] = v.raw[i];
}
}
do { } while (0);
return ret;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> Expand(Vec128<T, N> v, const Mask128<T, N> mask) {
size_t in_pos = 0;
Vec128<T, N> ret;
for (size_t i = 0; i < N; ++i) {
if (mask.bits[i]) {
ret.raw[i] = v.raw[in_pos++];
} else {
ret.raw[i] = T();
}
}
return ret;
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> LoadExpand(MFromD<D> mask, D d,
const TFromD<D>* __restrict__ unaligned) {
size_t in_pos = 0;
VFromD<D> ret;
for (size_t i = 0; i < Lanes(d); ++i) {
if (mask.bits[i]) {
ret.raw[i] = unaligned[in_pos++];
} else {
ret.raw[i] = TFromD<D>();
}
}
return ret;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> CompressNot(Vec128<T, N> v, Mask128<T, N> mask) {
size_t count = 0;
Vec128<T, N> ret;
for (size_t i = 0; i < N; ++i) {
if (!mask.bits[i]) {
ret.raw[count++] = v.raw[i];
}
}
for (size_t i = 0; i < N; ++i) {
if (mask.bits[i]) {
ret.raw[count++] = v.raw[i];
}
}
do { } while (0);
return ret;
}
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<uint64_t> CompressBlocksNot(Vec128<uint64_t> v,
Mask128<uint64_t> ) {
return v;
}
template <typename T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec128<T, N> CompressBits(Vec128<T, N> v,
const uint8_t* __restrict__ bits) {
return Compress(v, LoadMaskBits(Simd<T, N, 0>(), bits));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressStore(VFromD<D> v, MFromD<D> mask, D d,
TFromD<D>* __restrict__ unaligned) {
size_t count = 0;
for (size_t i = 0; i < MaxLanes(d); ++i) {
if (mask.bits[i]) {
unaligned[count++] = v.raw[i];
}
}
return count;
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressBlendedStore(VFromD<D> v, MFromD<D> mask, D d,
TFromD<D>* __restrict__ unaligned) {
return CompressStore(v, mask, d, unaligned);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) != (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressBitsStore(VFromD<D> v, const uint8_t* __restrict__ bits,
D d, TFromD<D>* __restrict__ unaligned) {
const MFromD<D> mask = LoadMaskBits(d, bits);
StoreU(Compress(v, mask), d, unaligned);
return CountTrue(d, mask);
}
template <class T>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, 1> SetAtOrAfterFirst(Mask128<T, 1> mask) { return mask; }
template <class T, size_t N, hwy::EnableIf<(N > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> SetAtOrAfterFirst(Mask128<T, N> mask) {
using TU = hwy::MakeUnsigned<T>;
Mask128<T, N> result;
TU result_lane_mask{0};
for (size_t i = 0; i < N; i++) {
result_lane_mask = static_cast<TU>(result_lane_mask | mask.bits[i]);
result.bits[i] = result_lane_mask;
}
return result;
}
template <class T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> SetBeforeFirst(Mask128<T, N> mask) {
return Not(SetAtOrAfterFirst(mask));
}
template <class T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> SetOnlyFirst(Mask128<T, N> mask) {
using TU = hwy::MakeUnsigned<T>;
using TI = hwy::MakeSigned<T>;
Mask128<T, N> result;
TU result_lane_mask = static_cast<TU>(~TU{0});
for (size_t i = 0; i < N; i++) {
const auto curr_lane_mask_bits = mask.bits[i];
result.bits[i] = static_cast<TU>(curr_lane_mask_bits & result_lane_mask);
result_lane_mask =
static_cast<TU>(result_lane_mask &
static_cast<TU>(-static_cast<TI>(mask.bits[i] == 0)));
}
return result;
}
template <class T, size_t N>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Mask128<T, N> SetAtOrBeforeFirst(Mask128<T, N> mask) {
using TU = hwy::MakeUnsigned<T>;
using TI = hwy::MakeSigned<T>;
Mask128<T, N> result;
TU result_lane_mask = static_cast<TU>(~TU{0});
for (size_t i = 0; i < N; i++) {
result.bits[i] = result_lane_mask;
result_lane_mask =
static_cast<TU>(result_lane_mask &
static_cast<TU>(-static_cast<TI>(mask.bits[i] == 0)));
}
return result;
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr, class VBF16>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> WidenMulPairwiseAdd(D df32, VBF16 a, VBF16 b) {
const Rebind<uint32_t, decltype(df32)> du32;
using VU32 = VFromD<decltype(du32)>;
const VU32 odd = Set(du32, 0xFFFF0000u);
const VU32 ae = ShiftLeft<16>(BitCast(du32, a));
const VU32 ao = And(BitCast(du32, a), odd);
const VU32 be = ShiftLeft<16>(BitCast(du32, b));
const VU32 bo = And(BitCast(du32, b), odd);
return Mul(BitCast(df32, ae), BitCast(df32, be)) +
Mul(BitCast(df32, ao), BitCast(df32, bo));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr, class VI16>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> WidenMulPairwiseAdd(D d32, VI16 a, VI16 b) {
using VI32 = VFromD<decltype(d32)>;
const VI32 ae = ShiftRight<16>(ShiftLeft<16>(BitCast(d32, a)));
const VI32 be = ShiftRight<16>(ShiftLeft<16>(BitCast(d32, b)));
const VI32 ao = ShiftRight<16>(BitCast(d32, a));
const VI32 bo = ShiftRight<16>(BitCast(d32, b));
return Add(Mul(ae, be), Mul(ao, bo));
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, float>()>* = nullptr, size_t N, class VBF16>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderWidenMulAccumulate(D df32, VBF16 a, VBF16 b,
const Vec128<float, N> sum0,
Vec128<float, N>& sum1) {
const Rebind<uint32_t, decltype(df32)> du32;
using VU32 = VFromD<decltype(du32)>;
const VU32 odd = Set(du32, 0xFFFF0000u);
const VU32 ae = ShiftLeft<16>(BitCast(du32, a));
const VU32 ao = And(BitCast(du32, a), odd);
const VU32 be = ShiftLeft<16>(BitCast(du32, b));
const VU32 bo = And(BitCast(du32, b), odd);
sum1 = MulAdd(BitCast(df32, ao), BitCast(df32, bo), sum1);
return MulAdd(BitCast(df32, ae), BitCast(df32, be), sum0);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, int32_t>()>* = nullptr, size_t N, class VI16>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> ReorderWidenMulAccumulate(D d32, VI16 a, VI16 b,
const Vec128<int32_t, N> sum0,
Vec128<int32_t, N>& sum1) {
using VI32 = VFromD<decltype(d32)>;
const VI32 ae = ShiftRight<16>(ShiftLeft<16>(BitCast(d32, a)));
const VI32 be = ShiftRight<16>(ShiftLeft<16>(BitCast(d32, b)));
const VI32 ao = ShiftRight<16>(BitCast(d32, a));
const VI32 bo = ShiftRight<16>(BitCast(d32, b));
sum1 = Add(Mul(ao, bo), sum1);
return Add(Mul(ae, be), sum0);
}
template <class VW>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VW RearrangeToOddPlusEven(VW sum0, VW sum1) {
return Add(sum0, sum1);
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SumOfLanes(D d, VFromD<D> v) {
T sum = T{0};
for (size_t i = 0; i < MaxLanes(d); ++i) {
sum += v.raw[i];
}
return Set(d, sum);
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) T ReduceSum(D d, VFromD<D> v) {
T sum = T{0};
for (size_t i = 0; i < MaxLanes(d); ++i) {
sum += v.raw[i];
}
return sum;
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> MinOfLanes(D d, VFromD<D> v) {
T min = HighestValue<T>();
for (size_t i = 0; i < MaxLanes(d); ++i) {
min = ((min) < (v.raw[i]) ? (min) : (v.raw[i]));
}
return Set(d, min);
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> MaxOfLanes(D d, VFromD<D> v) {
T max = LowestValue<T>();
for (size_t i = 0; i < MaxLanes(d); ++i) {
max = ((max) > (v.raw[i]) ? (max) : (v.raw[i]));
}
return Set(d, max);
}
inline __attribute__((always_inline)) Vec128<uint64_t> MulEven(Vec128<uint64_t> a, Vec128<uint64_t> b) {
alignas(16) uint64_t mul[2];
mul[0] = Mul128(GetLane(a), GetLane(b), &mul[1]);
return Load(Full128<uint64_t>(), mul);
}
inline __attribute__((always_inline)) Vec128<uint64_t> MulOdd(Vec128<uint64_t> a, Vec128<uint64_t> b) {
alignas(16) uint64_t mul[2];
const Half<Full128<uint64_t>> d2;
mul[0] =
Mul128(GetLane(UpperHalf(d2, a)), GetLane(UpperHalf(d2, b)), &mul[1]);
return Load(Full128<uint64_t>(), mul);
}
}
}
static_assert(true, "For requiring trailing semicolon");
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_EMU128 {
template <class V>
using LaneType = decltype(GetLane(V()));
template <class D>
using Vec = decltype(Zero(D()));
template <class D>
using Mask = decltype(MaskFromVec(Zero(D())));
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Clamp(const V v, const V lo, const V hi) {
return Min(Max(lo, v), hi);
}
template <size_t kLanes, class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> CombineShiftRightLanes(D d, VFromD<D> hi, VFromD<D> lo) {
constexpr size_t kBytes = kLanes * sizeof(TFromD<D>);
static_assert(kBytes < 16, "Shift count is per-block");
return CombineShiftRightBytes<kBytes>(d, hi, lo);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec<D> SignBit(D d) {
const RebindToUnsigned<decltype(d)> du;
return BitCast(d, Set(du, SignMask<TFromD<D>>()));
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec<D> NaN(D d) {
const RebindToSigned<D> di;
return BitCast(d, Set(di, LimitsMax<TFromD<decltype(di)>>()));
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec<D> Inf(D d) {
const RebindToUnsigned<D> du;
using T = TFromD<D>;
using TU = TFromD<decltype(du)>;
const TU max_x2 = static_cast<TU>(MaxExponentTimes2<T>());
return BitCast(d, Set(du, max_x2 >> 1));
}
template <class DTo, class DFrom>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DTo> ZeroExtendResizeBitCast(DTo d_to, DFrom d_from,
VFromD<DFrom> v) {
return detail::ZeroExtendResizeBitCast(hwy::SizeTag<d_from.MaxBytes()>(),
hwy::SizeTag<d_to.MaxBytes()>(), d_to,
d_from, v);
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void SafeFillN(const size_t num, const T value, D d,
T* __restrict__ to) {
(void)d;
for (size_t i = 0; i < num; ++i) {
to[i] = value;
}
}
template <class D, typename T = TFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void SafeCopyN(const size_t num, D d, const T* __restrict__ from,
T* __restrict__ to) {
(void)d;
for (size_t i = 0; i < num; ++i) {
to[i] = from[i];
}
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V BitwiseIfThenElse(V mask, V yes, V no) {
return Or(And(mask, yes), AndNot(mask, no));
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AbsDiff(V a, V b) {
return Sub(Max(a, b), Min(a, b));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint8_t>()>* = nullptr,
hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromD<DFromV<V> >) > ((((((1LL << 61) | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0)))) & -((((1LL << 61) | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0))))) == (1LL << 62) ? 0 : 4))>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) Vec<Repartition<uint64_t, DFromV<V>>> SumsOf8AbsDiff(V a, V b) {
return SumsOf8(AbsDiff(a, b));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedAdd(V a, V b) {
const DFromV<decltype(a)> d;
const auto sum = Add(a, b);
const auto overflow_mask =
MaskFromVec(BroadcastSignBit(AndNot(Xor(a, b), Xor(a, sum))));
const auto overflow_result =
Xor(BroadcastSignBit(a), Set(d, LimitsMax<int32_t>()));
return IfThenElse(overflow_mask, overflow_result, sum);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, int32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedSub(V a, V b) {
const DFromV<decltype(a)> d;
const auto diff = Sub(a, b);
const auto overflow_mask =
MaskFromVec(BroadcastSignBit(And(Xor(a, b), Xor(a, diff))));
const auto overflow_result =
Xor(BroadcastSignBit(a), Set(d, LimitsMax<int32_t>()));
return IfThenElse(overflow_mask, overflow_result, diff);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedAdd(V a, V b) {
const DFromV<decltype(a)> d;
const auto sum = Add(a, b);
const auto overflow_mask =
MaskFromVec(BroadcastSignBit(AndNot(Xor(a, b), Xor(a, sum))));
const auto overflow_result =
Xor(BroadcastSignBit(a), Set(d, LimitsMax<int64_t>()));
return IfThenElse(overflow_mask, overflow_result, sum);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, int64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedSub(V a, V b) {
const DFromV<decltype(a)> d;
const auto diff = Sub(a, b);
const auto overflow_mask =
MaskFromVec(BroadcastSignBit(And(Xor(a, b), Xor(a, diff))));
const auto overflow_result =
Xor(BroadcastSignBit(a), Set(d, LimitsMax<int64_t>()));
return IfThenElse(overflow_mask, overflow_result, diff);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedAdd(V a, V b) {
return Add(a, Min(b, Not(a)));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedSub(V a, V b) {
return Sub(a, Min(a, b));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedAdd(V a, V b) {
return Add(a, Min(b, Not(a)));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V SaturatedSub(V a, V b) {
return Sub(a, Min(a, b));
}
template <class DN, hwy::EnableIf<IsSigned<TFromD<DN> >() && !IsFloat<TFromD<DN> >() && !IsSpecialFloat<TFromD<DN> >()>* = nullptr, class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
class V2 = VFromD<Rebind<TFromV<V>, DN>>,
hwy::EnableIf<(sizeof(TFromD<DN>) < sizeof(TFromV<V>))>* = nullptr,
hwy::EnableIf<(DFromV<V>::kPrivateLanes == DFromV<V2>::kPrivateLanes)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DN> DemoteTo(DN dn, V v) {
const DFromV<decltype(v)> d;
const RebindToSigned<decltype(d)> di;
const RebindToUnsigned<decltype(dn)> dn_u;
const auto i2i_demote_result = DemoteTo(dn, BitCast(di, v));
const auto max_signed_val = Set(dn, hwy::HighestValue<TFromD<DN>>());
return BitCast(
dn, Min(BitCast(dn_u, i2i_demote_result), BitCast(dn_u, max_signed_val)));
}
template <class DN, hwy::EnableIf<IsSigned<TFromD<DN> >() && !IsFloat<TFromD<DN> >() && !IsSpecialFloat<TFromD<DN> >()>* = nullptr, class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
class V2 = VFromD<Repartition<TFromV<V>, DN>>,
hwy::EnableIf<sizeof(TFromV<V>) == (sizeof(TFromD<DN>) * 2)>* = nullptr,
hwy::EnableIf<(DFromV<V>::kPrivateLanes == DFromV<V2>::kPrivateLanes)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<DN> ReorderDemote2To(DN dn, V a, V b) {
const DFromV<decltype(a)> d;
const RebindToSigned<decltype(d)> di;
const RebindToUnsigned<decltype(dn)> dn_u;
const auto i2i_demote_result =
ReorderDemote2To(dn, BitCast(di, a), BitCast(di, b));
const auto max_signed_val = Set(dn, hwy::HighestValue<TFromD<DN>>());
return BitCast(
dn, Min(BitCast(dn_u, i2i_demote_result), BitCast(dn_u, max_signed_val)));
}
namespace detail {
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const RebindToFloat<decltype(d)> df;
const auto f32_bits = BitCast(d, ConvertTo(df, v));
return BitCast(d, ShiftRight<23>(f32_bits));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint32_t>()>* = nullptr>
inline __attribute__((always_inline)) V I32RangeU32ToF32BiasedExp(V v) {
const DFromV<decltype(v)> d;
const RebindToFloat<decltype(d)> df;
const RebindToUnsigned<decltype(d)> d_src;
const auto f32_bits = BitCast(d, ConvertTo(df, BitCast(d_src, v)));
return ShiftRight<23>(f32_bits);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes <= 16 / 4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Rebind<uint32_t, decltype(d)> du32;
const auto f32_biased_exp_as_u32 =
I32RangeU32ToF32BiasedExp(PromoteTo(du32, v));
return TruncateTo(d, f32_biased_exp_as_u32);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 16 / 4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Half<decltype(d)> dh;
const Rebind<uint32_t, decltype(dh)> du32;
const auto lo_u32 = PromoteTo(du32, LowerHalf(dh, v));
const auto hi_u32 = PromoteTo(du32, UpperHalf(dh, v));
const auto lo_f32_biased_exp_as_u32 = I32RangeU32ToF32BiasedExp(lo_u32);
const auto hi_f32_biased_exp_as_u32 = I32RangeU32ToF32BiasedExp(hi_u32);
return OrderedTruncate2To(d, lo_f32_biased_exp_as_u32,
hi_f32_biased_exp_as_u32);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes <= 16 / 4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Rebind<uint32_t, decltype(d)> du32;
const auto f32_biased_exp_as_u32 =
I32RangeU32ToF32BiasedExp(PromoteTo(du32, v));
return U8FromU32(f32_biased_exp_as_u32);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 16 / 4)>* = nullptr,
hwy::EnableIf<(D::kPrivateLanes <= 16 / 2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Half<decltype(d)> dh;
const Rebind<uint32_t, decltype(dh)> du32;
const Repartition<uint16_t, decltype(du32)> du16;
const auto lo_u32 = PromoteTo(du32, LowerHalf(dh, v));
const auto hi_u32 = PromoteTo(du32, UpperHalf(dh, v));
const auto lo_f32_biased_exp_as_u32 = I32RangeU32ToF32BiasedExp(lo_u32);
const auto hi_f32_biased_exp_as_u32 = I32RangeU32ToF32BiasedExp(hi_u32);
const auto f32_biased_exp_as_u16 = OrderedTruncate2To(
du16, lo_f32_biased_exp_as_u32, hi_f32_biased_exp_as_u32);
return TruncateTo(d, f32_biased_exp_as_u16);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 16 / 2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Half<decltype(d)> dh;
const Half<decltype(dh)> dq;
const Rebind<uint32_t, decltype(dq)> du32;
const Repartition<uint16_t, decltype(du32)> du16;
const auto lo_half = LowerHalf(dh, v);
const auto hi_half = UpperHalf(dh, v);
const auto u32_q0 = PromoteTo(du32, LowerHalf(dq, lo_half));
const auto u32_q1 = PromoteTo(du32, UpperHalf(dq, lo_half));
const auto u32_q2 = PromoteTo(du32, LowerHalf(dq, hi_half));
const auto u32_q3 = PromoteTo(du32, UpperHalf(dq, hi_half));
const auto f32_biased_exp_as_u32_q0 = I32RangeU32ToF32BiasedExp(u32_q0);
const auto f32_biased_exp_as_u32_q1 = I32RangeU32ToF32BiasedExp(u32_q1);
const auto f32_biased_exp_as_u32_q2 = I32RangeU32ToF32BiasedExp(u32_q2);
const auto f32_biased_exp_as_u32_q3 = I32RangeU32ToF32BiasedExp(u32_q3);
const auto lo_f32_biased_exp_as_u16 = OrderedTruncate2To(
du16, f32_biased_exp_as_u32_q0, f32_biased_exp_as_u32_q1);
const auto hi_f32_biased_exp_as_u16 = OrderedTruncate2To(
du16, f32_biased_exp_as_u32_q2, f32_biased_exp_as_u32_q3);
return OrderedTruncate2To(d, lo_f32_biased_exp_as_u16,
hi_f32_biased_exp_as_u16);
}
template <class D>
using F32ExpLzcntMinMaxRepartition =
Repartition<UnsignedFromSize<((sizeof(TFromD<D>)) < (4) ? (sizeof(TFromD<D>)) : (4))>, D>;
template <class V>
using F32ExpLzcntMinMaxCmpV = VFromD<F32ExpLzcntMinMaxRepartition<DFromV<V>>>;
template <class V>
inline __attribute__((always_inline)) F32ExpLzcntMinMaxCmpV<V> F32ExpLzcntMinMaxBitCast(V v) {
const DFromV<decltype(v)> d;
const F32ExpLzcntMinMaxRepartition<decltype(d)> d2;
return BitCast(d2, v);
}
template <class D, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>()>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> UIntToF32BiasedExp(D d, VFromD<D> v) {
const Repartition<uint32_t, decltype(d)> du32;
const auto f32_biased_exp = UIntToF32BiasedExp(du32, BitCast(du32, v));
const auto f32_biased_exp_adj =
IfThenZeroElse(Eq(f32_biased_exp, Zero(du32)),
BitCast(du32, Set(d, 0x0000002000000000u)));
const auto adj_f32_biased_exp = Add(f32_biased_exp, f32_biased_exp_adj);
return ShiftRight<32>(BitCast(
d, Max(F32ExpLzcntMinMaxBitCast(adj_f32_biased_exp),
F32ExpLzcntMinMaxBitCast(Reverse2(du32, adj_f32_biased_exp)))));
}
template <class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr>
inline __attribute__((always_inline)) V UIntToF32BiasedExp(V v) {
const DFromV<decltype(v)> d;
return UIntToF32BiasedExp(d, v);
}
template <class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2))) != 0>* = nullptr>
inline __attribute__((always_inline)) V NormalizeForUIntTruncConvToF32(V v) {
return v;
}
template <class V, hwy::EnableIf<!IsSigned<TFromV<V> >()>* = nullptr,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 4) | (1 << 8))) != 0>* = nullptr>
inline __attribute__((always_inline)) V NormalizeForUIntTruncConvToF32(V v) {
return AndNot(ShiftRight<24>(v), v);
}
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V HighestSetBitIndex(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
const auto f32_biased_exp = detail::UIntToF32BiasedExp(
detail::NormalizeForUIntTruncConvToF32(BitCast(du, v)));
return BitCast(d, Sub(f32_biased_exp, Set(du, TU{127})));
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V LeadingZeroCount(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
constexpr TU kNumOfBitsInT{sizeof(TU) * 8};
const auto f32_biased_exp = detail::UIntToF32BiasedExp(
detail::NormalizeForUIntTruncConvToF32(BitCast(du, v)));
const auto lz_count = Sub(Set(du, TU{kNumOfBitsInT + 126}), f32_biased_exp);
return BitCast(d,
Min(detail::F32ExpLzcntMinMaxBitCast(lz_count),
detail::F32ExpLzcntMinMaxBitCast(Set(du, kNumOfBitsInT))));
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V TrailingZeroCount(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const RebindToSigned<decltype(d)> di;
using TU = TFromD<decltype(du)>;
const auto vi = BitCast(di, v);
const auto lowest_bit = BitCast(du, And(vi, Neg(vi)));
constexpr TU kNumOfBitsInT{sizeof(TU) * 8};
const auto f32_biased_exp = detail::UIntToF32BiasedExp(lowest_bit);
const auto tz_count = Sub(f32_biased_exp, Set(du, TU{127}));
return BitCast(d,
Min(detail::F32ExpLzcntMinMaxBitCast(tz_count),
detail::F32ExpLzcntMinMaxBitCast(Set(du, kNumOfBitsInT))));
}
namespace detail {
template <class V>
inline __attribute__((always_inline)) V SubBytesMulInverseAndAffineLookup(V state, V affine_tblL,
V affine_tblU) {
const DFromV<V> du;
const auto mask = Set(du, uint8_t{0xF});
{
alignas(16) static constexpr uint8_t basisL[16] = {
0x00, 0x70, 0x2A, 0x5A, 0x98, 0xE8, 0xB2, 0xC2,
0x08, 0x78, 0x22, 0x52, 0x90, 0xE0, 0xBA, 0xCA};
alignas(16) static constexpr uint8_t basisU[16] = {
0x00, 0x4D, 0x7C, 0x31, 0x7D, 0x30, 0x01, 0x4C,
0x81, 0xCC, 0xFD, 0xB0, 0xFC, 0xB1, 0x80, 0xCD};
const auto sL = And(state, mask);
const auto sU = ShiftRight<4>(state);
const auto gf4L = TableLookupBytes(LoadDup128(du, basisL), sL);
const auto gf4U = TableLookupBytes(LoadDup128(du, basisU), sU);
state = Xor(gf4L, gf4U);
}
alignas(16) static constexpr uint8_t kZetaInv[16] = {
0x80, 7, 11, 15, 6, 10, 4, 1, 9, 8, 5, 2, 12, 14, 13, 3};
alignas(16) static constexpr uint8_t kInv[16] = {
0x80, 1, 8, 13, 15, 6, 5, 14, 2, 12, 11, 10, 9, 3, 7, 4};
const auto tbl = LoadDup128(du, kInv);
const auto sL = And(state, mask);
const auto sU = ShiftRight<4>(state);
const auto sX = Xor(sU, sL);
const auto invL = TableLookupBytes(LoadDup128(du, kZetaInv), sL);
const auto invU = TableLookupBytes(tbl, sU);
const auto invX = TableLookupBytes(tbl, sX);
const auto outL = Xor(sX, TableLookupBytesOr0(tbl, Xor(invL, invU)));
const auto outU = Xor(sU, TableLookupBytesOr0(tbl, Xor(invL, invX)));
const auto affL = TableLookupBytesOr0(affine_tblL, outL);
const auto affU = TableLookupBytesOr0(affine_tblU, outU);
return Xor(affL, affU);
}
template <class V>
inline __attribute__((always_inline)) V SubBytes(V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t kAffineL[16] = {
0x00, 0xC7, 0xBD, 0x6F, 0x17, 0x6D, 0xD2, 0xD0,
0x78, 0xA8, 0x02, 0xC5, 0x7A, 0xBF, 0xAA, 0x15};
alignas(16) static constexpr uint8_t kAffineU[16] = {
0x00, 0x6A, 0xBB, 0x5F, 0xA5, 0x74, 0xE4, 0xCF,
0xFA, 0x35, 0x2B, 0x41, 0xD1, 0x90, 0x1E, 0x8E};
return Xor(SubBytesMulInverseAndAffineLookup(state, LoadDup128(du, kAffineL),
LoadDup128(du, kAffineU)),
Set(du, uint8_t{0x63}));
}
template <class V>
inline __attribute__((always_inline)) V InvSubBytes(V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t kGF2P4InvToGF2P8InvL[16]{
0x00, 0x40, 0xF9, 0x7E, 0x53, 0xEA, 0x87, 0x13,
0x2D, 0x3E, 0x94, 0xD4, 0xB9, 0x6D, 0xAA, 0xC7};
alignas(16) static constexpr uint8_t kGF2P4InvToGF2P8InvU[16]{
0x00, 0x1D, 0x44, 0x93, 0x0F, 0x56, 0xD7, 0x12,
0x9C, 0x8E, 0xC5, 0xD8, 0x59, 0x81, 0x4B, 0xCA};
const auto b = Xor(Xor3(Or(ShiftLeft<1>(state), ShiftRight<7>(state)),
Or(ShiftLeft<3>(state), ShiftRight<5>(state)),
Or(ShiftLeft<6>(state), ShiftRight<2>(state))),
Set(du, uint8_t{0x05}));
return SubBytesMulInverseAndAffineLookup(
b, LoadDup128(du, kGF2P4InvToGF2P8InvL),
LoadDup128(du, kGF2P4InvToGF2P8InvU));
}
}
namespace detail {
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ShiftRows(const V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t kShiftRow[16] = {
0, 5, 10, 15,
4, 9, 14, 3,
8, 13, 2, 7,
12, 1, 6, 11};
const auto shift_row = LoadDup128(du, kShiftRow);
return TableLookupBytes(state, shift_row);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V InvShiftRows(const V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t kShiftRow[16] = {
0, 13, 10, 7,
4, 1, 14, 11,
8, 5, 2, 15,
12, 9, 6, 3};
const auto shift_row = LoadDup128(du, kShiftRow);
return TableLookupBytes(state, shift_row);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V GF2P8Mod11BMulBy2(V v) {
const DFromV<V> du;
const RebindToSigned<decltype(du)> di;
const auto msb = Lt(BitCast(di, v), Zero(di));
const auto overflow = BitCast(du, IfThenElseZero(msb, Set(di, int8_t{0x1B})));
return Xor(Add(v, v), overflow);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V MixColumns(const V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t k2301[16] = {
2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13};
alignas(16) static constexpr uint8_t k1230[16] = {
1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12};
const auto d = GF2P8Mod11BMulBy2(state);
const auto s2301 = TableLookupBytes(state, LoadDup128(du, k2301));
const auto d_s2301 = Xor(d, s2301);
const auto t_s2301 = Xor(state, d_s2301);
const auto t1230_s3012 = TableLookupBytes(t_s2301, LoadDup128(du, k1230));
return Xor(d_s2301, t1230_s3012);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V InvMixColumns(const V state) {
const DFromV<V> du;
alignas(16) static constexpr uint8_t k2301[16] = {
2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13};
alignas(16) static constexpr uint8_t k1230[16] = {
1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12};
const auto v1230 = LoadDup128(du, k1230);
const auto sx2 = GF2P8Mod11BMulBy2(state);
const auto sx4 = GF2P8Mod11BMulBy2(sx2);
const auto sx8 = GF2P8Mod11BMulBy2(sx4);
const auto sx9 = Xor(sx8, state);
const auto sx11 = Xor(sx9, sx2);
const auto sx13 = Xor(sx9, sx4);
const auto sx14 = Xor3(sx8, sx4, sx2);
const auto sx13_0123_sx9_1230 = Xor(sx13, TableLookupBytes(sx9, v1230));
const auto sx14_0123_sx11_1230 = Xor(sx14, TableLookupBytes(sx11, v1230));
const auto sx13_2301_sx9_3012 =
TableLookupBytes(sx13_0123_sx9_1230, LoadDup128(du, k2301));
return Xor(sx14_0123_sx11_1230, sx13_2301_sx9_3012);
}
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESRound(V state, const V round_key) {
state = detail::SubBytes(state);
state = detail::ShiftRows(state);
state = detail::MixColumns(state);
state = Xor(state, round_key);
return state;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESLastRound(V state, const V round_key) {
state = detail::SubBytes(state);
state = detail::ShiftRows(state);
state = Xor(state, round_key);
return state;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESInvMixColumns(V state) {
return detail::InvMixColumns(state);
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESRoundInv(V state, const V round_key) {
state = detail::InvSubBytes(state);
state = detail::InvShiftRows(state);
state = detail::InvMixColumns(state);
state = Xor(state, round_key);
return state;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESLastRoundInv(V state, const V round_key) {
state = detail::InvSubBytes(state);
state = detail::InvShiftRows(state);
state = Xor(state, round_key);
return state;
}
template <uint8_t kRcon, class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, uint8_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V AESKeyGenAssist(V v) {
alignas(16) static constexpr uint8_t kRconXorMask[16] = {
0, 0, 0, 0, kRcon, 0, 0, 0, 0, 0, 0, 0, kRcon, 0, 0, 0};
alignas(16) static constexpr uint8_t kRotWordShuffle[16] = {
4, 5, 6, 7, 5, 6, 7, 4, 12, 13, 14, 15, 13, 14, 15, 12};
const DFromV<decltype(v)> d;
const auto sub_word_result = detail::SubBytes(v);
const auto rot_word_result =
TableLookupBytes(sub_word_result, LoadDup128(d, kRotWordShuffle));
return Xor(rot_word_result, LoadDup128(d, kRconXorMask));
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CLMulLower(V a, V b) {
const DFromV<V> d;
static_assert(IsSame<TFromD<decltype(d)>, uint64_t>(), "V must be u64");
const auto k1 = Set(d, 0x1111111111111111ULL);
const auto k2 = Set(d, 0x2222222222222222ULL);
const auto k4 = Set(d, 0x4444444444444444ULL);
const auto k8 = Set(d, 0x8888888888888888ULL);
const auto a0 = And(a, k1);
const auto a1 = And(a, k2);
const auto a2 = And(a, k4);
const auto a3 = And(a, k8);
const auto b0 = And(b, k1);
const auto b1 = And(b, k2);
const auto b2 = And(b, k4);
const auto b3 = And(b, k8);
auto m0 = Xor(MulEven(a0, b0), MulEven(a1, b3));
auto m1 = Xor(MulEven(a0, b1), MulEven(a1, b0));
auto m2 = Xor(MulEven(a0, b2), MulEven(a1, b1));
auto m3 = Xor(MulEven(a0, b3), MulEven(a1, b2));
m0 = Xor(m0, Xor(MulEven(a2, b2), MulEven(a3, b1)));
m1 = Xor(m1, Xor(MulEven(a2, b3), MulEven(a3, b2)));
m2 = Xor(m2, Xor(MulEven(a2, b0), MulEven(a3, b3)));
m3 = Xor(m3, Xor(MulEven(a2, b1), MulEven(a3, b0)));
return Or(Or(And(m0, k1), And(m1, k2)), Or(And(m2, k4), And(m3, k8)));
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CLMulUpper(V a, V b) {
const DFromV<V> d;
static_assert(IsSame<TFromD<decltype(d)>, uint64_t>(), "V must be u64");
const auto k1 = Set(d, 0x1111111111111111ULL);
const auto k2 = Set(d, 0x2222222222222222ULL);
const auto k4 = Set(d, 0x4444444444444444ULL);
const auto k8 = Set(d, 0x8888888888888888ULL);
const auto a0 = And(a, k1);
const auto a1 = And(a, k2);
const auto a2 = And(a, k4);
const auto a3 = And(a, k8);
const auto b0 = And(b, k1);
const auto b1 = And(b, k2);
const auto b2 = And(b, k4);
const auto b3 = And(b, k8);
auto m0 = Xor(MulOdd(a0, b0), MulOdd(a1, b3));
auto m1 = Xor(MulOdd(a0, b1), MulOdd(a1, b0));
auto m2 = Xor(MulOdd(a0, b2), MulOdd(a1, b1));
auto m3 = Xor(MulOdd(a0, b3), MulOdd(a1, b2));
m0 = Xor(m0, Xor(MulOdd(a2, b2), MulOdd(a3, b1)));
m1 = Xor(m1, Xor(MulOdd(a2, b3), MulOdd(a3, b2)));
m2 = Xor(m2, Xor(MulOdd(a2, b0), MulOdd(a3, b3)));
m3 = Xor(m3, Xor(MulOdd(a2, b1), MulOdd(a3, b0)));
return Or(Or(And(m0, k1), And(m1, k2)), Or(And(m2, k4), And(m3, k8)));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr,
hwy::EnableIf<(D::kPrivateLanes * sizeof(TFromD<D>) > 8)>* = nullptr, void* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
alignas(16) constexpr uint8_t kLookup[16] = {
0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
};
const auto lo = And(v, Set(d, uint8_t{0xF}));
const auto hi = ShiftRight<4>(v);
const auto lookup = LoadDup128(d, kLookup);
return Add(TableLookupBytes(lookup, hi), TableLookupBytes(lookup, lo));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint8_t>()>* = nullptr,
hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 8>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
const V k33 = Set(d, uint8_t{0x33});
v = Sub(v, And(ShiftRight<1>(v), Set(d, uint8_t{0x55})));
v = Add(And(ShiftRight<2>(v), k33), And(v, k33));
return And(Add(v, ShiftRight<4>(v)), Set(d, uint8_t{0x0F}));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint16_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
const Repartition<uint8_t, decltype(d)> d8;
const auto vals = BitCast(d, PopulationCount(BitCast(d8, v)));
return Add(ShiftRight<8>(vals), And(vals, Set(d, uint16_t{0xFF})));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint32_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
Repartition<uint16_t, decltype(d)> d16;
auto vals = BitCast(d, PopulationCount(BitCast(d16, v)));
return Add(ShiftRight<16>(vals), And(vals, Set(d, uint32_t{0xFF})));
}
template <class V, class D = DFromV<V>, hwy::EnableIf<IsSame<TFromD<D>, uint64_t>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V PopulationCount(V v) {
const D d;
Repartition<uint32_t, decltype(d)> d32;
auto vals = BitCast(d, PopulationCount(BitCast(d32, v)));
return Add(ShiftRight<32>(vals), And(vals, Set(d, 0xFFULL)));
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V MulAdd(V mul, V x, V add) {
return Add(Mul(mul, x), add);
}
template <class V, hwy::EnableIf<!hwy::IsFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V NegMulAdd(V mul, V x, V add) {
return Sub(add, Mul(mul, x));
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ApproximateReciprocal(V v) {
const DFromV<decltype(v)> d;
return Div(Set(d, 1.0), v);
}
template <class V, hwy::EnableIf<IsSame<TFromD<DFromV<V> >, double>()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ApproximateReciprocalSqrt(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const auto half = Mul(v, Set(d, 0.5));
const auto guess = BitCast(d, Sub(Set(du, uint64_t{0x5FE6EB50C7B537A9u}),
ShiftRight<1>(BitCast(du, v))));
return Mul(guess, NegMulAdd(Mul(half, guess), guess, Set(d, 1.5)));
}
template <class V, class D, typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressBitsStore(V v, const uint8_t* __restrict__ bits, D d,
T* unaligned) {
alignas(16) T lanes[MaxLanes(d)];
Store(v, d, lanes);
const Simd<T, ((MaxLanes(d)) < (8) ? (MaxLanes(d)) : (8)), 0> d8;
T* __restrict__ pos = unaligned;
alignas(16) constexpr T table[2048] = {
0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
1, 0, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
2, 0, 1, 3, 4, 5, 6, 7, 0, 2, 1, 3, 4, 5, 6, 7,
1, 2, 0, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
3, 0, 1, 2, 4, 5, 6, 7, 0, 3, 1, 2, 4, 5, 6, 7,
1, 3, 0, 2, 4, 5, 6, 7, 0, 1, 3, 2, 4, 5, 6, 7,
2, 3, 0, 1, 4, 5, 6, 7, 0, 2, 3, 1, 4, 5, 6, 7,
1, 2, 3, 0, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
4, 0, 1, 2, 3, 5, 6, 7, 0, 4, 1, 2, 3, 5, 6, 7,
1, 4, 0, 2, 3, 5, 6, 7, 0, 1, 4, 2, 3, 5, 6, 7,
2, 4, 0, 1, 3, 5, 6, 7, 0, 2, 4, 1, 3, 5, 6, 7,
1, 2, 4, 0, 3, 5, 6, 7, 0, 1, 2, 4, 3, 5, 6, 7,
3, 4, 0, 1, 2, 5, 6, 7, 0, 3, 4, 1, 2, 5, 6, 7,
1, 3, 4, 0, 2, 5, 6, 7, 0, 1, 3, 4, 2, 5, 6, 7,
2, 3, 4, 0, 1, 5, 6, 7, 0, 2, 3, 4, 1, 5, 6, 7,
1, 2, 3, 4, 0, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
5, 0, 1, 2, 3, 4, 6, 7, 0, 5, 1, 2, 3, 4, 6, 7,
1, 5, 0, 2, 3, 4, 6, 7, 0, 1, 5, 2, 3, 4, 6, 7,
2, 5, 0, 1, 3, 4, 6, 7, 0, 2, 5, 1, 3, 4, 6, 7,
1, 2, 5, 0, 3, 4, 6, 7, 0, 1, 2, 5, 3, 4, 6, 7,
3, 5, 0, 1, 2, 4, 6, 7, 0, 3, 5, 1, 2, 4, 6, 7,
1, 3, 5, 0, 2, 4, 6, 7, 0, 1, 3, 5, 2, 4, 6, 7,
2, 3, 5, 0, 1, 4, 6, 7, 0, 2, 3, 5, 1, 4, 6, 7,
1, 2, 3, 5, 0, 4, 6, 7, 0, 1, 2, 3, 5, 4, 6, 7,
4, 5, 0, 1, 2, 3, 6, 7, 0, 4, 5, 1, 2, 3, 6, 7,
1, 4, 5, 0, 2, 3, 6, 7, 0, 1, 4, 5, 2, 3, 6, 7,
2, 4, 5, 0, 1, 3, 6, 7, 0, 2, 4, 5, 1, 3, 6, 7,
1, 2, 4, 5, 0, 3, 6, 7, 0, 1, 2, 4, 5, 3, 6, 7,
3, 4, 5, 0, 1, 2, 6, 7, 0, 3, 4, 5, 1, 2, 6, 7,
1, 3, 4, 5, 0, 2, 6, 7, 0, 1, 3, 4, 5, 2, 6, 7,
2, 3, 4, 5, 0, 1, 6, 7, 0, 2, 3, 4, 5, 1, 6, 7,
1, 2, 3, 4, 5, 0, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
6, 0, 1, 2, 3, 4, 5, 7, 0, 6, 1, 2, 3, 4, 5, 7,
1, 6, 0, 2, 3, 4, 5, 7, 0, 1, 6, 2, 3, 4, 5, 7,
2, 6, 0, 1, 3, 4, 5, 7, 0, 2, 6, 1, 3, 4, 5, 7,
1, 2, 6, 0, 3, 4, 5, 7, 0, 1, 2, 6, 3, 4, 5, 7,
3, 6, 0, 1, 2, 4, 5, 7, 0, 3, 6, 1, 2, 4, 5, 7,
1, 3, 6, 0, 2, 4, 5, 7, 0, 1, 3, 6, 2, 4, 5, 7,
2, 3, 6, 0, 1, 4, 5, 7, 0, 2, 3, 6, 1, 4, 5, 7,
1, 2, 3, 6, 0, 4, 5, 7, 0, 1, 2, 3, 6, 4, 5, 7,
4, 6, 0, 1, 2, 3, 5, 7, 0, 4, 6, 1, 2, 3, 5, 7,
1, 4, 6, 0, 2, 3, 5, 7, 0, 1, 4, 6, 2, 3, 5, 7,
2, 4, 6, 0, 1, 3, 5, 7, 0, 2, 4, 6, 1, 3, 5, 7,
1, 2, 4, 6, 0, 3, 5, 7, 0, 1, 2, 4, 6, 3, 5, 7,
3, 4, 6, 0, 1, 2, 5, 7, 0, 3, 4, 6, 1, 2, 5, 7,
1, 3, 4, 6, 0, 2, 5, 7, 0, 1, 3, 4, 6, 2, 5, 7,
2, 3, 4, 6, 0, 1, 5, 7, 0, 2, 3, 4, 6, 1, 5, 7,
1, 2, 3, 4, 6, 0, 5, 7, 0, 1, 2, 3, 4, 6, 5, 7,
5, 6, 0, 1, 2, 3, 4, 7, 0, 5, 6, 1, 2, 3, 4, 7,
1, 5, 6, 0, 2, 3, 4, 7, 0, 1, 5, 6, 2, 3, 4, 7,
2, 5, 6, 0, 1, 3, 4, 7, 0, 2, 5, 6, 1, 3, 4, 7,
1, 2, 5, 6, 0, 3, 4, 7, 0, 1, 2, 5, 6, 3, 4, 7,
3, 5, 6, 0, 1, 2, 4, 7, 0, 3, 5, 6, 1, 2, 4, 7,
1, 3, 5, 6, 0, 2, 4, 7, 0, 1, 3, 5, 6, 2, 4, 7,
2, 3, 5, 6, 0, 1, 4, 7, 0, 2, 3, 5, 6, 1, 4, 7,
1, 2, 3, 5, 6, 0, 4, 7, 0, 1, 2, 3, 5, 6, 4, 7,
4, 5, 6, 0, 1, 2, 3, 7, 0, 4, 5, 6, 1, 2, 3, 7,
1, 4, 5, 6, 0, 2, 3, 7, 0, 1, 4, 5, 6, 2, 3, 7,
2, 4, 5, 6, 0, 1, 3, 7, 0, 2, 4, 5, 6, 1, 3, 7,
1, 2, 4, 5, 6, 0, 3, 7, 0, 1, 2, 4, 5, 6, 3, 7,
3, 4, 5, 6, 0, 1, 2, 7, 0, 3, 4, 5, 6, 1, 2, 7,
1, 3, 4, 5, 6, 0, 2, 7, 0, 1, 3, 4, 5, 6, 2, 7,
2, 3, 4, 5, 6, 0, 1, 7, 0, 2, 3, 4, 5, 6, 1, 7,
1, 2, 3, 4, 5, 6, 0, 7, 0, 1, 2, 3, 4, 5, 6, 7,
7, 0, 1, 2, 3, 4, 5, 6, 0, 7, 1, 2, 3, 4, 5, 6,
1, 7, 0, 2, 3, 4, 5, 6, 0, 1, 7, 2, 3, 4, 5, 6,
2, 7, 0, 1, 3, 4, 5, 6, 0, 2, 7, 1, 3, 4, 5, 6,
1, 2, 7, 0, 3, 4, 5, 6, 0, 1, 2, 7, 3, 4, 5, 6,
3, 7, 0, 1, 2, 4, 5, 6, 0, 3, 7, 1, 2, 4, 5, 6,
1, 3, 7, 0, 2, 4, 5, 6, 0, 1, 3, 7, 2, 4, 5, 6,
2, 3, 7, 0, 1, 4, 5, 6, 0, 2, 3, 7, 1, 4, 5, 6,
1, 2, 3, 7, 0, 4, 5, 6, 0, 1, 2, 3, 7, 4, 5, 6,
4, 7, 0, 1, 2, 3, 5, 6, 0, 4, 7, 1, 2, 3, 5, 6,
1, 4, 7, 0, 2, 3, 5, 6, 0, 1, 4, 7, 2, 3, 5, 6,
2, 4, 7, 0, 1, 3, 5, 6, 0, 2, 4, 7, 1, 3, 5, 6,
1, 2, 4, 7, 0, 3, 5, 6, 0, 1, 2, 4, 7, 3, 5, 6,
3, 4, 7, 0, 1, 2, 5, 6, 0, 3, 4, 7, 1, 2, 5, 6,
1, 3, 4, 7, 0, 2, 5, 6, 0, 1, 3, 4, 7, 2, 5, 6,
2, 3, 4, 7, 0, 1, 5, 6, 0, 2, 3, 4, 7, 1, 5, 6,
1, 2, 3, 4, 7, 0, 5, 6, 0, 1, 2, 3, 4, 7, 5, 6,
5, 7, 0, 1, 2, 3, 4, 6, 0, 5, 7, 1, 2, 3, 4, 6,
1, 5, 7, 0, 2, 3, 4, 6, 0, 1, 5, 7, 2, 3, 4, 6,
2, 5, 7, 0, 1, 3, 4, 6, 0, 2, 5, 7, 1, 3, 4, 6,
1, 2, 5, 7, 0, 3, 4, 6, 0, 1, 2, 5, 7, 3, 4, 6,
3, 5, 7, 0, 1, 2, 4, 6, 0, 3, 5, 7, 1, 2, 4, 6,
1, 3, 5, 7, 0, 2, 4, 6, 0, 1, 3, 5, 7, 2, 4, 6,
2, 3, 5, 7, 0, 1, 4, 6, 0, 2, 3, 5, 7, 1, 4, 6,
1, 2, 3, 5, 7, 0, 4, 6, 0, 1, 2, 3, 5, 7, 4, 6,
4, 5, 7, 0, 1, 2, 3, 6, 0, 4, 5, 7, 1, 2, 3, 6,
1, 4, 5, 7, 0, 2, 3, 6, 0, 1, 4, 5, 7, 2, 3, 6,
2, 4, 5, 7, 0, 1, 3, 6, 0, 2, 4, 5, 7, 1, 3, 6,
1, 2, 4, 5, 7, 0, 3, 6, 0, 1, 2, 4, 5, 7, 3, 6,
3, 4, 5, 7, 0, 1, 2, 6, 0, 3, 4, 5, 7, 1, 2, 6,
1, 3, 4, 5, 7, 0, 2, 6, 0, 1, 3, 4, 5, 7, 2, 6,
2, 3, 4, 5, 7, 0, 1, 6, 0, 2, 3, 4, 5, 7, 1, 6,
1, 2, 3, 4, 5, 7, 0, 6, 0, 1, 2, 3, 4, 5, 7, 6,
6, 7, 0, 1, 2, 3, 4, 5, 0, 6, 7, 1, 2, 3, 4, 5,
1, 6, 7, 0, 2, 3, 4, 5, 0, 1, 6, 7, 2, 3, 4, 5,
2, 6, 7, 0, 1, 3, 4, 5, 0, 2, 6, 7, 1, 3, 4, 5,
1, 2, 6, 7, 0, 3, 4, 5, 0, 1, 2, 6, 7, 3, 4, 5,
3, 6, 7, 0, 1, 2, 4, 5, 0, 3, 6, 7, 1, 2, 4, 5,
1, 3, 6, 7, 0, 2, 4, 5, 0, 1, 3, 6, 7, 2, 4, 5,
2, 3, 6, 7, 0, 1, 4, 5, 0, 2, 3, 6, 7, 1, 4, 5,
1, 2, 3, 6, 7, 0, 4, 5, 0, 1, 2, 3, 6, 7, 4, 5,
4, 6, 7, 0, 1, 2, 3, 5, 0, 4, 6, 7, 1, 2, 3, 5,
1, 4, 6, 7, 0, 2, 3, 5, 0, 1, 4, 6, 7, 2, 3, 5,
2, 4, 6, 7, 0, 1, 3, 5, 0, 2, 4, 6, 7, 1, 3, 5,
1, 2, 4, 6, 7, 0, 3, 5, 0, 1, 2, 4, 6, 7, 3, 5,
3, 4, 6, 7, 0, 1, 2, 5, 0, 3, 4, 6, 7, 1, 2, 5,
1, 3, 4, 6, 7, 0, 2, 5, 0, 1, 3, 4, 6, 7, 2, 5,
2, 3, 4, 6, 7, 0, 1, 5, 0, 2, 3, 4, 6, 7, 1, 5,
1, 2, 3, 4, 6, 7, 0, 5, 0, 1, 2, 3, 4, 6, 7, 5,
5, 6, 7, 0, 1, 2, 3, 4, 0, 5, 6, 7, 1, 2, 3, 4,
1, 5, 6, 7, 0, 2, 3, 4, 0, 1, 5, 6, 7, 2, 3, 4,
2, 5, 6, 7, 0, 1, 3, 4, 0, 2, 5, 6, 7, 1, 3, 4,
1, 2, 5, 6, 7, 0, 3, 4, 0, 1, 2, 5, 6, 7, 3, 4,
3, 5, 6, 7, 0, 1, 2, 4, 0, 3, 5, 6, 7, 1, 2, 4,
1, 3, 5, 6, 7, 0, 2, 4, 0, 1, 3, 5, 6, 7, 2, 4,
2, 3, 5, 6, 7, 0, 1, 4, 0, 2, 3, 5, 6, 7, 1, 4,
1, 2, 3, 5, 6, 7, 0, 4, 0, 1, 2, 3, 5, 6, 7, 4,
4, 5, 6, 7, 0, 1, 2, 3, 0, 4, 5, 6, 7, 1, 2, 3,
1, 4, 5, 6, 7, 0, 2, 3, 0, 1, 4, 5, 6, 7, 2, 3,
2, 4, 5, 6, 7, 0, 1, 3, 0, 2, 4, 5, 6, 7, 1, 3,
1, 2, 4, 5, 6, 7, 0, 3, 0, 1, 2, 4, 5, 6, 7, 3,
3, 4, 5, 6, 7, 0, 1, 2, 0, 3, 4, 5, 6, 7, 1, 2,
1, 3, 4, 5, 6, 7, 0, 2, 0, 1, 3, 4, 5, 6, 7, 2,
2, 3, 4, 5, 6, 7, 0, 1, 0, 2, 3, 4, 5, 6, 7, 1,
1, 2, 3, 4, 5, 6, 7, 0, 0, 1, 2, 3, 4, 5, 6, 7};
for (size_t i = 0; i < Lanes(d); i += 8) {
const size_t bits8 = bits[i / 8];
const auto indices = Load(d8, table + bits8 * 8);
const auto compressed = TableLookupBytes(LoadU(d8, lanes + i), indices);
StoreU(compressed, d8, pos);
pos += PopCount(bits8);
}
return static_cast<size_t>(pos - unaligned);
}
template <class V, class M, class D, typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressStore(V v, M mask, D d, T* __restrict__ unaligned) {
uint8_t bits[((size_t{8}) > (MaxLanes(d) / 8) ? (size_t{8}) : (MaxLanes(d) / 8))];
(void)StoreMaskBits(d, mask, bits);
return CompressBitsStore(v, bits, d, unaligned);
}
template <class V, class M, class D, typename T, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t CompressBlendedStore(V v, M mask, D d,
T* __restrict__ unaligned) {
alignas(16) T buf[MaxLanes(d)];
const size_t bytes = CompressStore(v, mask, d, buf);
BlendedStore(Load(d, buf), FirstN(d, bytes), d, unaligned);
return bytes;
}
template <class V, class M, typename T = TFromV<V>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Compress(V v, const M mask) {
const DFromV<V> d;
alignas(16) T lanes[MaxLanes(d)];
(void)CompressStore(v, mask, d, lanes);
return Load(d, lanes);
}
template <class V, typename T = TFromV<V>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CompressBits(V v, const uint8_t* __restrict__ bits) {
const DFromV<V> d;
alignas(16) T lanes[MaxLanes(d)];
(void)CompressBitsStore(v, bits, d, lanes);
return Load(d, lanes);
}
template <class V, class M, typename T = TFromV<V>, hwy::EnableIf<sizeof(T) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V CompressNot(V v, M mask) {
return Compress(v, Not(mask));
}
template <class D>
using IndicesFromD = decltype(IndicesFromVec(D(), Zero(RebindToUnsigned<D>())));
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> TwoTablesLookupLanes(D , VFromD<D> a, VFromD<D> b,
IndicesFromD<D> idx) {
return TwoTablesLookupLanes(a, b, idx);
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseLaneBytes(V v) {
const DFromV<V> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, Reverse2(du8, BitCast(du8, v)));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (4)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseLaneBytes(V v) {
const DFromV<V> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, Reverse4(du8, BitCast(du8, v)));
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (8)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseLaneBytes(V v) {
const DFromV<V> d;
const Repartition<uint8_t, decltype(d)> du8;
return BitCast(d, Reverse8(du8, BitCast(du8, v)));
}
namespace detail {
template <int kShiftAmt, int kShrResultMask, class V,
hwy::EnableIf<(DFromV<V>::kPrivateLanes * sizeof(TFromD<DFromV<V> >) > 1 - 1)>* = nullptr>
inline __attribute__((always_inline)) V UI8ReverseBitsStep(V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const RebindToUnsigned<decltype(d)> d_shift;
const auto v_to_shift = BitCast(d_shift, v);
const auto shl_result = BitCast(d, ShiftLeft<kShiftAmt>(v_to_shift));
const auto shr_result = BitCast(d, ShiftRight<kShiftAmt>(v_to_shift));
const auto shr_result_mask =
BitCast(d, Set(du, static_cast<uint8_t>(kShrResultMask)));
return Or(And(shr_result, shr_result_mask),
AndNot(shr_result_mask, shl_result));
}
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseBits(V v) {
auto result = detail::UI8ReverseBitsStep<1, 0x55>(v);
result = detail::UI8ReverseBitsStep<2, 0x33>(result);
result = detail::UI8ReverseBitsStep<4, 0x0F>(result);
return result;
}
template <class V, hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 2) | (1 << 4) | (1 << 8))) != 0>* = nullptr,
hwy::EnableIf<!hwy::IsFloat<TFromV<V> >() && !hwy::IsSpecialFloat<TFromV<V> >()>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ReverseBits(V v) {
const DFromV<decltype(v)> d;
const Repartition<uint8_t, decltype(d)> du8;
return ReverseLaneBytes(BitCast(d, ReverseBits(BitCast(du8, v))));
}
namespace detail {
template <class D>
inline __attribute__((always_inline)) Vec<D> Per4LaneBlkShufDupSet4xU32(D d, const uint32_t x3,
const uint32_t x2,
const uint32_t x1,
const uint32_t x0) {
alignas(16) const uint32_t lanes[4] = {x0, x1, x2, x3};
constexpr size_t kMaxBytes = d.MaxBytes();
constexpr size_t kMinLanesToLoad = 4;
constexpr size_t kNumToLoad =
((kMaxBytes / sizeof(uint32_t)) > (kMinLanesToLoad) ? (kMaxBytes / sizeof(uint32_t)) : (kMinLanesToLoad));
const CappedTag<uint32_t, kNumToLoad> d_load;
return ResizeBitCast(d, LoadDup128(d_load, lanes));
}
}
namespace detail {
template <class V>
inline __attribute__((always_inline)) V Per2LaneBlockShuffle(hwy::SizeTag<0> , V v) {
return DupEven(v);
}
template <class V>
inline __attribute__((always_inline)) V Per2LaneBlockShuffle(hwy::SizeTag<1> , V v) {
const DFromV<decltype(v)> d;
return Reverse2(d, v);
}
template <class V>
inline __attribute__((always_inline)) V Per2LaneBlockShuffle(hwy::SizeTag<2> , V v) {
return v;
}
template <class V>
inline __attribute__((always_inline)) V Per2LaneBlockShuffle(hwy::SizeTag<3> , V v) {
return DupOdd(v);
}
inline __attribute__((always_inline)) uint32_t U8x4Per4LaneBlkIndices(const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
return static_cast<uint32_t>((idx3 << 24) | (idx2 << 16) | (idx1 << 8) |
idx0);
}
template <class D>
inline __attribute__((always_inline)) Vec<D> TblLookupPer4LaneBlkU8IdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const Repartition<uint32_t, D> du32;
return ResizeBitCast(
d, Set(du32, U8x4Per4LaneBlkIndices(idx3, idx2, idx1, idx0)));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (1)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> TblLookupPer4LaneBlkIdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
return TblLookupPer4LaneBlkU8IdxInBlk(d, idx3, idx2, idx1, idx0);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (2)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> TblLookupPer4LaneBlkIdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const uint16_t u16_idx0 = static_cast<uint16_t>(idx0);
const uint16_t u16_idx1 = static_cast<uint16_t>(idx1);
const uint16_t u16_idx2 = static_cast<uint16_t>(idx2);
const uint16_t u16_idx3 = static_cast<uint16_t>(idx3);
alignas(16)
const uint16_t indices[8] = {u16_idx0, u16_idx1, u16_idx2, u16_idx3,
u16_idx0, u16_idx1, u16_idx2, u16_idx3};
constexpr size_t kMinLanesToLoad = 8;
constexpr size_t kNumToLoad = ((D::kPrivateLanes) > (kMinLanesToLoad) ? (D::kPrivateLanes) : (kMinLanesToLoad));
const CappedTag<uint16_t, kNumToLoad> d_load;
return ResizeBitCast(d, LoadDup128(d_load, indices));
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (4)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> TblLookupPer4LaneBlkIdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
return Per4LaneBlkShufDupSet4xU32(d, idx3, idx2, idx1, idx0);
}
template <class D, hwy::EnableIf<sizeof(TFromD<D>) == (8)>* = nullptr>
inline __attribute__((always_inline)) VFromD<D> TblLookupPer4LaneBlkIdxInBlk(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const RebindToUnsigned<decltype(d)> du;
const Rebind<uint32_t, decltype(d)> du32;
return BitCast(d, PromoteTo(du, Per4LaneBlkShufDupSet4xU32(du32, idx3, idx2,
idx1, idx0)));
}
template <class D, void* = nullptr>
inline __attribute__((always_inline)) IndicesFromD<D> TblLookupPer4LaneBlkShufIdx(D d, const uint32_t idx3,
const uint32_t idx2,
const uint32_t idx1,
const uint32_t idx0) {
const RebindToUnsigned<decltype(d)> du;
using TU = TFromD<decltype(du)>;
auto idx_in_blk = TblLookupPer4LaneBlkIdxInBlk(du, idx3, idx2, idx1, idx0);
constexpr size_t kN = D::kPrivateLanes;
if (kN < 4) {
idx_in_blk = And(idx_in_blk, Set(du, static_cast<TU>(kN - 1)));
}
const auto blk_offsets =
And(Iota(du, TU{0}), Set(du, static_cast<TU>(~TU{3})));
return IndicesFromVec(d, Add(idx_in_blk, blk_offsets));
}
template <class V, void* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlkShufDoTblLookup(V v, IndicesFromD<DFromV<V>> idx) {
return TableLookupLanes(v, idx);
}
template <class V>
inline __attribute__((always_inline)) V TblLookupPer4LaneBlkShuf(V v, size_t idx3210) {
const DFromV<decltype(v)> d;
const uint32_t idx3 = static_cast<uint32_t>((idx3210 >> 6) & 3);
const uint32_t idx2 = static_cast<uint32_t>((idx3210 >> 4) & 3);
const uint32_t idx1 = static_cast<uint32_t>((idx3210 >> 2) & 3);
const uint32_t idx0 = static_cast<uint32_t>(idx3210 & 3);
const auto idx = TblLookupPer4LaneBlkShufIdx(d, idx3, idx2, idx1, idx0);
return Per4LaneBlkShufDoTblLookup(v, idx);
}
template <size_t kIdx3210, size_t kLaneSize, size_t kVectSize, class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> ,
hwy::SizeTag<kLaneSize> ,
hwy::SizeTag<kVectSize> ,
V v) {
return TblLookupPer4LaneBlkShuf(v, kIdx3210);
}
template <class V>
inline __attribute__((always_inline)) VFromD<RepartitionToWide<DFromV<V>>> Per4LaneBlockShufCastToWide(
hwy::FloatTag , hwy::SizeTag<4> , V v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> dw;
return BitCast(dw, v);
}
template <size_t kLaneSize, class V>
inline __attribute__((always_inline)) VFromD<RepartitionToWide<RebindToUnsigned<DFromV<V>>>>
Per4LaneBlockShufCastToWide(hwy::FloatTag ,
hwy::SizeTag<kLaneSize> , V v) {
const DFromV<decltype(v)> d;
const RebindToUnsigned<decltype(d)> du;
const RepartitionToWide<decltype(du)> dw;
return BitCast(dw, v);
}
template <size_t kLaneSize, class V>
inline __attribute__((always_inline)) VFromD<RepartitionToWide<DFromV<V>>> Per4LaneBlockShufCastToWide(
hwy::NonFloatTag ,
hwy::SizeTag<kLaneSize> , V v) {
const DFromV<decltype(v)> d;
const RepartitionToWide<decltype(d)> dw;
return BitCast(dw, v);
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x1B> , V v) {
const DFromV<decltype(v)> d;
return Reverse4(d, v);
}
template <class V,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2) | (1 ? (1 << 4) : 0))) != 0>* = nullptr
>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x44> , V v) {
const DFromV<decltype(v)> d;
const auto vw = Per4LaneBlockShufCastToWide(
hwy::IsFloatTag<TFromV<V>>(), hwy::SizeTag<sizeof(TFromV<V>)>(), v);
return BitCast(d, DupEven(vw));
}
template <class V,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2) | (1 ? (1 << 4) : 0))) != 0>* = nullptr
>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x4E> , V v) {
const DFromV<decltype(v)> d;
const auto vw = Per4LaneBlockShufCastToWide(
hwy::IsFloatTag<TFromV<V>>(), hwy::SizeTag<sizeof(TFromV<V>)>(), v);
const DFromV<decltype(vw)> dw;
return BitCast(d, Reverse2(dw, vw));
}
template <class V, hwy::EnableIf<(DFromV<V>::kPrivateLanes == 4)>* = nullptr,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2))) != 0>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x50> , V v) {
const DFromV<decltype(v)> d;
return InterleaveLower(d, v, v);
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (4)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x50> , V v) {
const DFromV<decltype(v)> d;
return InterleaveLower(d, v, v);
}
template <class V, hwy::EnableIf<(DFromV<V>::kPrivateLanes == 4)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0x88> , V v) {
const DFromV<decltype(v)> d;
return ConcatEven(d, v, v);
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xA0> , V v) {
return DupEven(v);
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xB1> , V v) {
const DFromV<decltype(v)> d;
return Reverse2(d, v);
}
template <class V, hwy::EnableIf<(DFromV<V>::kPrivateLanes == 4)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xDD> , V v) {
const DFromV<decltype(v)> d;
return ConcatOdd(d, v, v);
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xE4> , V v) {
return v;
}
template <class V,
hwy::EnableIf<((size_t{1} << sizeof(TFromV<V>)) & ((1 << 1) | (1 << 2) | (1 ? (1 << 4) : 0))) != 0>* = nullptr
>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xEE> , V v) {
const DFromV<decltype(v)> d;
const auto vw = Per4LaneBlockShufCastToWide(
hwy::IsFloatTag<TFromV<V>>(), hwy::SizeTag<sizeof(TFromV<V>)>(), v);
return BitCast(d, DupOdd(vw));
}
template <class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xF5> , V v) {
return DupOdd(v);
}
template <class V, hwy::EnableIf<sizeof(TFromV<V>) == (4)>* = nullptr>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<0xFA> , V v) {
const DFromV<decltype(v)> d;
return InterleaveUpper(d, v, v);
}
template <size_t kIdx3210, class V>
inline __attribute__((always_inline)) V Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210> idx_3210_tag, V v) {
const DFromV<decltype(v)> d;
return Per4LaneBlockShuffle(idx_3210_tag, hwy::SizeTag<sizeof(TFromV<V>)>(),
hwy::SizeTag<d.MaxBytes()>(), v);
}
}
template <size_t kIdx3, size_t kIdx2, size_t kIdx1, size_t kIdx0, class V,
hwy::EnableIf<(DFromV<V>::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Per4LaneBlockShuffle(V v) {
static_assert(kIdx0 <= 3, "kIdx0 <= 3 must be true");
static_assert(kIdx1 <= 3, "kIdx1 <= 3 must be true");
static_assert(kIdx2 <= 3, "kIdx2 <= 3 must be true");
static_assert(kIdx3 <= 3, "kIdx3 <= 3 must be true");
return v;
}
template <size_t kIdx3, size_t kIdx2, size_t kIdx1, size_t kIdx0, class V,
hwy::EnableIf<(DFromV<V>::kPrivateLanes == 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Per4LaneBlockShuffle(V v) {
static_assert(kIdx0 <= 3, "kIdx0 <= 3 must be true");
static_assert(kIdx1 <= 3, "kIdx1 <= 3 must be true");
static_assert(kIdx2 <= 3, "kIdx2 <= 3 must be true");
static_assert(kIdx3 <= 3, "kIdx3 <= 3 must be true");
constexpr bool isReverse2 = (kIdx0 == 1 || kIdx1 == 0) && (kIdx0 != kIdx1);
constexpr size_t kPer2BlkIdx0 = (kIdx0 <= 1) ? kIdx0 : (isReverse2 ? 1 : 0);
constexpr size_t kPer2BlkIdx1 = (kIdx1 <= 1) ? kIdx1 : (isReverse2 ? 0 : 1);
constexpr size_t kIdx10 = (kPer2BlkIdx1 << 1) | kPer2BlkIdx0;
static_assert(kIdx10 <= 3, "kIdx10 <= 3 must be true");
return detail::Per2LaneBlockShuffle(hwy::SizeTag<kIdx10>(), v);
}
template <size_t kIdx3, size_t kIdx2, size_t kIdx1, size_t kIdx0, class V,
hwy::EnableIf<(DFromV<V>::kPrivateLanes > 2)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Per4LaneBlockShuffle(V v) {
static_assert(kIdx0 <= 3, "kIdx0 <= 3 must be true");
static_assert(kIdx1 <= 3, "kIdx1 <= 3 must be true");
static_assert(kIdx2 <= 3, "kIdx2 <= 3 must be true");
static_assert(kIdx3 <= 3, "kIdx3 <= 3 must be true");
constexpr size_t kIdx3210 =
(kIdx3 << 6) | (kIdx2 << 4) | (kIdx1 << 2) | kIdx0;
return detail::Per4LaneBlockShuffle(hwy::SizeTag<kIdx3210>(), v);
}
template <class D>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) size_t Blocks(D d) {
return (d.MaxBytes() <= 16) ? 1 : ((Lanes(d) * sizeof(TFromD<D>) + 15) / 16);
}
template <int kBlockIdx, class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V InsertBlock(V , V blk_to_insert) {
static_assert(kBlockIdx == 0, "Invalid block index");
return blk_to_insert;
}
template <int kBlockIdx, class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V ExtractBlock(V v) {
static_assert(kBlockIdx == 0, "Invalid block index");
return v;
}
template <int kBlockIdx, class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V BroadcastBlock(V v) {
static_assert(kBlockIdx == 0, "Invalid block index");
return v;
}
template<int kLane, class V, hwy::EnableIf<DFromV<V>::kPrivateLanes * sizeof(TFromV<V>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V BroadcastLane(V v) {
return Broadcast<kLane>(v);
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Slide1Up(D d, VFromD<D> ) {
return Zero(d);
}
template <class D, hwy::EnableIf<(D::kPrivateLanes == 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Slide1Down(D d, VFromD<D> ) {
return Zero(d);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Slide1Up(D d, VFromD<D> v) {
return ShiftLeftLanes<1>(d, v);
}
template <class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr, hwy::EnableIf<(D::kPrivateLanes > 1)>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> Slide1Down(D d, VFromD<D> v) {
return ShiftRightLanes<1>(d, v);
}
template <int kBlocks, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideUpBlocks(D , VFromD<D> v) {
static_assert(kBlocks == 0, "kBlocks == 0 must be true");
return v;
}
template <int kBlocks, class D, hwy::EnableIf<D::kPrivateLanes * sizeof(TFromD<D>) <= 16>* = nullptr>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) VFromD<D> SlideDownBlocks(D , VFromD<D> v) {
static_assert(kBlocks == 0, "kBlocks == 0 must be true");
return v;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Add(V a, V b) {
return a + b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Sub(V a, V b) {
return a - b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Mul(V a, V b) {
return a * b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) V Div(V a, V b) {
return a / b;
}
template <class V>
V Shl(V a, V b) {
return a << b;
}
template <class V>
V Shr(V a, V b) {
return a >> b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Eq(V a, V b) -> decltype(a == b) {
return a == b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Ne(V a, V b) -> decltype(a == b) {
return a != b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Lt(V a, V b) -> decltype(a == b) {
return a < b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Gt(V a, V b) -> decltype(a == b) {
return a > b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Ge(V a, V b) -> decltype(a == b) {
return a >= b;
}
template <class V>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) auto Le(V a, V b) -> decltype(a == b) {
return a <= b;
}
}
}
static_assert(true, "For requiring trailing semicolon");
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_EMU128 {
template <class D, class V>
inline __attribute__((always_inline)) V Acos(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAcos(const D d, VecArg<V> x) {
return Acos(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Acosh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAcosh(const D d, VecArg<V> x) {
return Acosh(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Asin(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAsin(const D d, VecArg<V> x) {
return Asin(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Asinh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAsinh(const D d, VecArg<V> x) {
return Asinh(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Atan(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAtan(const D d, VecArg<V> x) {
return Atan(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Atanh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallAtanh(const D d, VecArg<V> x) {
return Atanh(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Cos(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallCos(const D d, VecArg<V> x) {
return Cos(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Exp(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallExp(const D d, VecArg<V> x) {
return Exp(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Expm1(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallExpm1(const D d, VecArg<V> x) {
return Expm1(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallLog(const D d, VecArg<V> x) {
return Log(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log10(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallLog10(const D d, VecArg<V> x) {
return Log10(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log1p(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallLog1p(const D d, VecArg<V> x) {
return Log1p(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log2(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallLog2(const D d, VecArg<V> x) {
return Log2(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Sin(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallSin(const D d, VecArg<V> x) {
return Sin(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Sinh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallSinh(const D d, VecArg<V> x) {
return Sinh(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Tanh(const D d, V x);
template <class D, class V>
__attribute__((noinline)) V CallTanh(const D d, VecArg<V> x) {
return Tanh(d, x);
}
namespace impl {
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1) {
return MulAdd(c1, x, c0);
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2) {
T x2 = Mul(x, x);
return MulAdd(x2, c2, MulAdd(c1, x, c0));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3) {
T x2 = Mul(x, x);
return MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
return MulAdd(x4, c4, MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
return MulAdd(x4, MulAdd(c5, x, c4),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
return MulAdd(x4, MulAdd(x2, c6, MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
return MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8, c8,
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8, MulAdd(c9, x, c8),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8, MulAdd(x2, c10, MulAdd(c9, x, c8)),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8, MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8)),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(
x8, MulAdd(x4, c12, MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8,
MulAdd(x4, MulAdd(c13, x, c12),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8,
MulAdd(x4, MulAdd(x2, c14, MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14, T c15) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
return MulAdd(x8,
MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14, T c15, T c16) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
T x16 = Mul(x8, x8);
return MulAdd(
x16, c16,
MulAdd(x8,
MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14, T c15, T c16, T c17) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
T x16 = Mul(x8, x8);
return MulAdd(
x16, MulAdd(c17, x, c16),
MulAdd(x8,
MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))));
}
template <class T>
inline __attribute__((always_inline)) __attribute__((unused)) T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5,
T c6, T c7, T c8, T c9, T c10, T c11,
T c12, T c13, T c14, T c15, T c16, T c17,
T c18) {
T x2 = Mul(x, x);
T x4 = Mul(x2, x2);
T x8 = Mul(x4, x4);
T x16 = Mul(x8, x8);
return MulAdd(
x16, MulAdd(x2, c18, MulAdd(c17, x, c16)),
MulAdd(x8,
MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)),
MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))),
MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)),
MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))));
}
template <class FloatOrDouble>
struct AsinImpl {};
template <class FloatOrDouble>
struct AtanImpl {};
template <class FloatOrDouble>
struct CosSinImpl {};
template <class FloatOrDouble>
struct ExpImpl {};
template <class FloatOrDouble>
struct LogImpl {};
template <>
struct AsinImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) V AsinPoly(D d, V x2, V ) {
const auto k0 = Set(d, +0.1666677296f);
const auto k1 = Set(d, +0.07495029271f);
const auto k2 = Set(d, +0.04547423869f);
const auto k3 = Set(d, +0.02424046025f);
const auto k4 = Set(d, +0.04197454825f);
return Estrin(x2, k0, k1, k2, k3, k4);
}
};
template <>
struct AsinImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) V AsinPoly(D d, V x2, V ) {
const auto k0 = Set(d, +0.1666666666666497543);
const auto k1 = Set(d, +0.07500000000378581611);
const auto k2 = Set(d, +0.04464285681377102438);
const auto k3 = Set(d, +0.03038195928038132237);
const auto k4 = Set(d, +0.02237176181932048341);
const auto k5 = Set(d, +0.01735956991223614604);
const auto k6 = Set(d, +0.01388715184501609218);
const auto k7 = Set(d, +0.01215360525577377331);
const auto k8 = Set(d, +0.006606077476277170610);
const auto k9 = Set(d, +0.01929045477267910674);
const auto k10 = Set(d, -0.01581918243329996643);
const auto k11 = Set(d, +0.03161587650653934628);
return Estrin(x2, k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11);
}
};
template <>
struct AtanImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) V AtanPoly(D d, V x) {
const auto k0 = Set(d, -0.333331018686294555664062f);
const auto k1 = Set(d, +0.199926957488059997558594f);
const auto k2 = Set(d, -0.142027363181114196777344f);
const auto k3 = Set(d, +0.106347933411598205566406f);
const auto k4 = Set(d, -0.0748900920152664184570312f);
const auto k5 = Set(d, +0.0425049886107444763183594f);
const auto k6 = Set(d, -0.0159569028764963150024414f);
const auto k7 = Set(d, +0.00282363896258175373077393f);
const auto y = Mul(x, x);
return MulAdd(Estrin(y, k0, k1, k2, k3, k4, k5, k6, k7), Mul(y, x), x);
}
};
template <>
struct AtanImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) V AtanPoly(D d, V x) {
const auto k0 = Set(d, -0.333333333333311110369124);
const auto k1 = Set(d, +0.199999999996591265594148);
const auto k2 = Set(d, -0.14285714266771329383765);
const auto k3 = Set(d, +0.111111105648261418443745);
const auto k4 = Set(d, -0.090908995008245008229153);
const auto k5 = Set(d, +0.0769219538311769618355029);
const auto k6 = Set(d, -0.0666573579361080525984562);
const auto k7 = Set(d, +0.0587666392926673580854313);
const auto k8 = Set(d, -0.0523674852303482457616113);
const auto k9 = Set(d, +0.0466667150077840625632675);
const auto k10 = Set(d, -0.0407629191276836500001934);
const auto k11 = Set(d, +0.0337852580001353069993897);
const auto k12 = Set(d, -0.0254517624932312641616861);
const auto k13 = Set(d, +0.016599329773529201970117);
const auto k14 = Set(d, -0.00889896195887655491740809);
const auto k15 = Set(d, +0.00370026744188713119232403);
const auto k16 = Set(d, -0.00110611831486672482563471);
const auto k17 = Set(d, +0.000209850076645816976906797);
const auto k18 = Set(d, -1.88796008463073496563746e-5);
const auto y = Mul(x, x);
return MulAdd(Estrin(y, k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11,
k12, k13, k14, k15, k16, k17, k18),
Mul(y, x), x);
}
};
template <>
struct CosSinImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> ToInt32(D , V x) {
return ConvertTo(Rebind<int32_t, D>(), x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Poly(D d, V x) {
const auto k0 = Set(d, -1.66666597127914428710938e-1f);
const auto k1 = Set(d, +8.33307858556509017944336e-3f);
const auto k2 = Set(d, -1.981069071916863322258e-4f);
const auto k3 = Set(d, +2.6083159809786593541503e-6f);
const auto y = Mul(x, x);
return MulAdd(Estrin(y, k0, k1, k2, k3), Mul(y, x), x);
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V CosReduce(D d, V x, VI32 q) {
const V kHalfPiPart0f = Set(d, -0.5f * 3.140625f);
const V kHalfPiPart1f = Set(d, -0.5f * 0.0009670257568359375f);
const V kHalfPiPart2f = Set(d, -0.5f * 6.2771141529083251953e-7f);
const V kHalfPiPart3f = Set(d, -0.5f * 1.2154201256553420762e-10f);
const V qf = ConvertTo(d, q);
x = MulAdd(qf, kHalfPiPart0f, x);
x = MulAdd(qf, kHalfPiPart1f, x);
x = MulAdd(qf, kHalfPiPart2f, x);
x = MulAdd(qf, kHalfPiPart3f, x);
return x;
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V SinReduce(D d, V x, VI32 q) {
const V kPiPart0f = Set(d, -3.140625f);
const V kPiPart1f = Set(d, -0.0009670257568359375f);
const V kPiPart2f = Set(d, -6.2771141529083251953e-7f);
const V kPiPart3f = Set(d, -1.2154201256553420762e-10f);
const V qf = ConvertTo(d, q);
x = MulAdd(qf, kPiPart0f, x);
x = MulAdd(qf, kPiPart1f, x);
x = MulAdd(qf, kPiPart2f, x);
x = MulAdd(qf, kPiPart3f, x);
return x;
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<Rebind<float, D>> CosSignFromQuadrant(D d, VI32 q) {
const VI32 kTwo = Set(Rebind<int32_t, D>(), 2);
return BitCast(d, ShiftLeft<30>(AndNot(q, kTwo)));
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<Rebind<float, D>> SinSignFromQuadrant(D d, VI32 q) {
const VI32 kOne = Set(Rebind<int32_t, D>(), 1);
return BitCast(d, ShiftLeft<31>(And(q, kOne)));
}
};
template <>
struct CosSinImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> ToInt32(D , V x) {
return DemoteTo(Rebind<int32_t, D>(), x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Poly(D d, V x) {
const auto k0 = Set(d, -0.166666666666666657414808);
const auto k1 = Set(d, +0.00833333333333332974823815);
const auto k2 = Set(d, -0.000198412698412696162806809);
const auto k3 = Set(d, +2.75573192239198747630416e-6);
const auto k4 = Set(d, -2.50521083763502045810755e-8);
const auto k5 = Set(d, +1.60590430605664501629054e-10);
const auto k6 = Set(d, -7.64712219118158833288484e-13);
const auto k7 = Set(d, +2.81009972710863200091251e-15);
const auto k8 = Set(d, -7.97255955009037868891952e-18);
const auto y = Mul(x, x);
return MulAdd(Estrin(y, k0, k1, k2, k3, k4, k5, k6, k7, k8), Mul(y, x), x);
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V CosReduce(D d, V x, VI32 q) {
const V kHalfPiPart0d = Set(d, -0.5 * 3.1415926218032836914);
const V kHalfPiPart1d = Set(d, -0.5 * 3.1786509424591713469e-8);
const V kHalfPiPart2d = Set(d, -0.5 * 1.2246467864107188502e-16);
const V kHalfPiPart3d = Set(d, -0.5 * 1.2736634327021899816e-24);
const V qf = PromoteTo(d, q);
x = MulAdd(qf, kHalfPiPart0d, x);
x = MulAdd(qf, kHalfPiPart1d, x);
x = MulAdd(qf, kHalfPiPart2d, x);
x = MulAdd(qf, kHalfPiPart3d, x);
return x;
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V SinReduce(D d, V x, VI32 q) {
const V kPiPart0d = Set(d, -3.1415926218032836914);
const V kPiPart1d = Set(d, -3.1786509424591713469e-8);
const V kPiPart2d = Set(d, -1.2246467864107188502e-16);
const V kPiPart3d = Set(d, -1.2736634327021899816e-24);
const V qf = PromoteTo(d, q);
x = MulAdd(qf, kPiPart0d, x);
x = MulAdd(qf, kPiPart1d, x);
x = MulAdd(qf, kPiPart2d, x);
x = MulAdd(qf, kPiPart3d, x);
return x;
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<Rebind<double, D>> CosSignFromQuadrant(D d, VI32 q) {
const VI32 kTwo = Set(Rebind<int32_t, D>(), 2);
return BitCast(
d, ShiftLeft<62>(PromoteTo(Rebind<int64_t, D>(), AndNot(q, kTwo))));
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<Rebind<double, D>> SinSignFromQuadrant(D d, VI32 q) {
const VI32 kOne = Set(Rebind<int32_t, D>(), 1);
return BitCast(
d, ShiftLeft<63>(PromoteTo(Rebind<int64_t, D>(), And(q, kOne))));
}
};
template <>
struct ExpImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> ToInt32(D , V x) {
return ConvertTo(Rebind<int32_t, D>(), x);
}
template <class D, class V>
inline __attribute__((always_inline)) V ExpPoly(D d, V x) {
const auto k0 = Set(d, +0.5f);
const auto k1 = Set(d, +0.166666671633720397949219f);
const auto k2 = Set(d, +0.0416664853692054748535156f);
const auto k3 = Set(d, +0.00833336077630519866943359f);
const auto k4 = Set(d, +0.00139304355252534151077271f);
const auto k5 = Set(d, +0.000198527617612853646278381f);
return MulAdd(Estrin(x, k0, k1, k2, k3, k4, k5), Mul(x, x), x);
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<D> Pow2I(D d, VI32 x) {
const Rebind<int32_t, D> di32;
const VI32 kOffset = Set(di32, 0x7F);
return BitCast(d, ShiftLeft<23>(Add(x, kOffset)));
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V LoadExpShortRange(D d, V x, VI32 e) {
const VI32 y = ShiftRight<1>(e);
return Mul(Mul(x, Pow2I(d, y)), Pow2I(d, Sub(e, y)));
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V ExpReduce(D d, V x, VI32 q) {
const V kLn2Part0f = Set(d, -0.693145751953125f);
const V kLn2Part1f = Set(d, -1.428606765330187045e-6f);
const V qf = ConvertTo(d, q);
x = MulAdd(qf, kLn2Part0f, x);
x = MulAdd(qf, kLn2Part1f, x);
return x;
}
};
template <>
struct LogImpl<float> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> Log2p1NoSubnormal(D , V x) {
const Rebind<int32_t, D> di32;
const Rebind<uint32_t, D> du32;
const auto kBias = Set(di32, 0x7F);
return Sub(BitCast(di32, ShiftRight<23>(BitCast(du32, x))), kBias);
}
template <class D, class V>
inline __attribute__((always_inline)) V LogPoly(D d, V x) {
const V k0 = Set(d, 0.66666662693f);
const V k1 = Set(d, 0.40000972152f);
const V k2 = Set(d, 0.28498786688f);
const V k3 = Set(d, 0.24279078841f);
const V x2 = Mul(x, x);
const V x4 = Mul(x2, x2);
return MulAdd(MulAdd(k2, x4, k0), x2, Mul(MulAdd(k3, x4, k1), x4));
}
};
template <>
struct ExpImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int32_t, D>> ToInt32(D , V x) {
return DemoteTo(Rebind<int32_t, D>(), x);
}
template <class D, class V>
inline __attribute__((always_inline)) V ExpPoly(D d, V x) {
const auto k0 = Set(d, +0.5);
const auto k1 = Set(d, +0.166666666666666851703837);
const auto k2 = Set(d, +0.0416666666666665047591422);
const auto k3 = Set(d, +0.00833333333331652721664984);
const auto k4 = Set(d, +0.00138888888889774492207962);
const auto k5 = Set(d, +0.000198412698960509205564975);
const auto k6 = Set(d, +2.4801587159235472998791e-5);
const auto k7 = Set(d, +2.75572362911928827629423e-6);
const auto k8 = Set(d, +2.75573911234900471893338e-7);
const auto k9 = Set(d, +2.51112930892876518610661e-8);
const auto k10 = Set(d, +2.08860621107283687536341e-9);
return MulAdd(Estrin(x, k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10),
Mul(x, x), x);
}
template <class D, class VI32>
inline __attribute__((always_inline)) Vec<D> Pow2I(D d, VI32 x) {
const Rebind<int32_t, D> di32;
const Rebind<int64_t, D> di64;
const VI32 kOffset = Set(di32, 0x3FF);
return BitCast(d, ShiftLeft<52>(PromoteTo(di64, Add(x, kOffset))));
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V LoadExpShortRange(D d, V x, VI32 e) {
const VI32 y = ShiftRight<1>(e);
return Mul(Mul(x, Pow2I(d, y)), Pow2I(d, Sub(e, y)));
}
template <class D, class V, class VI32>
inline __attribute__((always_inline)) V ExpReduce(D d, V x, VI32 q) {
const V kLn2Part0d = Set(d, -0.6931471805596629565116018);
const V kLn2Part1d = Set(d, -0.28235290563031577122588448175e-12);
const V qf = PromoteTo(d, q);
x = MulAdd(qf, kLn2Part0d, x);
x = MulAdd(qf, kLn2Part1d, x);
return x;
}
};
template <>
struct LogImpl<double> {
template <class D, class V>
inline __attribute__((always_inline)) Vec<Rebind<int64_t, D>> Log2p1NoSubnormal(D , V x) {
const Rebind<int64_t, D> di64;
const Rebind<uint64_t, D> du64;
return Sub(BitCast(di64, ShiftRight<52>(BitCast(du64, x))),
Set(di64, 0x3FF));
}
template <class D, class V>
inline __attribute__((always_inline)) V LogPoly(D d, V x) {
const V k0 = Set(d, 0.6666666666666735130);
const V k1 = Set(d, 0.3999999999940941908);
const V k2 = Set(d, 0.2857142874366239149);
const V k3 = Set(d, 0.2222219843214978396);
const V k4 = Set(d, 0.1818357216161805012);
const V k5 = Set(d, 0.1531383769920937332);
const V k6 = Set(d, 0.1479819860511658591);
const V x2 = Mul(x, x);
const V x4 = Mul(x2, x2);
return MulAdd(MulAdd(MulAdd(MulAdd(k6, x4, k4), x4, k2), x4, k0), x2,
(Mul(MulAdd(MulAdd(k5, x4, k3), x4, k1), x4)));
}
};
template <class D, class V, bool kAllowSubnormals = true>
inline __attribute__((always_inline)) V Log(const D d, V x) {
using T = TFromD<D>;
impl::LogImpl<T> impl;
constexpr bool kIsF32 = (sizeof(T) == 4);
const V kLn2Hi = Set(d, kIsF32 ? static_cast<T>(0.69313812256f)
: static_cast<T>(0.693147180369123816490));
const V kLn2Lo = Set(d, kIsF32 ? static_cast<T>(9.0580006145e-6f)
: static_cast<T>(1.90821492927058770002e-10));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kMinNormal = Set(d, kIsF32 ? static_cast<T>(1.175494351e-38f)
: static_cast<T>(2.2250738585072014e-308));
const V kScale = Set(d, kIsF32 ? static_cast<T>(3.355443200e+7f)
: static_cast<T>(1.8014398509481984e+16));
using TI = MakeSigned<T>;
const Rebind<TI, D> di;
using VI = decltype(Zero(di));
const VI kLowerBits = Set(di, kIsF32 ? static_cast<TI>(0x00000000L)
: static_cast<TI>(0xFFFFFFFFLL));
const VI kMagic = Set(di, kIsF32 ? static_cast<TI>(0x3F3504F3L)
: static_cast<TI>(0x3FE6A09E00000000LL));
const VI kExpMask = Set(di, kIsF32 ? static_cast<TI>(0x3F800000L)
: static_cast<TI>(0x3FF0000000000000LL));
const VI kExpScale =
Set(di, kIsF32 ? static_cast<TI>(-25) : static_cast<TI>(-54));
const VI kManMask = Set(di, kIsF32 ? static_cast<TI>(0x7FFFFFL)
: static_cast<TI>(0xFFFFF00000000LL));
VI exp_bits;
V exp;
if (kAllowSubnormals == true) {
const auto is_denormal = Lt(x, kMinNormal);
x = IfThenElse(is_denormal, Mul(x, kScale), x);
exp_bits = Add(BitCast(di, x), Sub(kExpMask, kMagic));
const VI exp_scale =
BitCast(di, IfThenElseZero(is_denormal, BitCast(d, kExpScale)));
exp = ConvertTo(
d, Add(exp_scale, impl.Log2p1NoSubnormal(d, BitCast(d, exp_bits))));
} else {
exp_bits = Add(BitCast(di, x), Sub(kExpMask, kMagic));
exp = ConvertTo(d, impl.Log2p1NoSubnormal(d, BitCast(d, exp_bits)));
}
const V y = Or(And(x, BitCast(d, kLowerBits)),
BitCast(d, Add(And(exp_bits, kManMask), kMagic)));
const V ym1 = Sub(y, kOne);
const V z = Div(ym1, Add(y, kOne));
return MulSub(
exp, kLn2Hi,
Sub(MulSub(z, Sub(ym1, impl.LogPoly(d, z)), Mul(exp, kLn2Lo)), ym1));
}
}
template <class D, class V>
inline __attribute__((always_inline)) V Acos(const D d, V x) {
using T = TFromD<D>;
const V kZero = Zero(d);
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kPi = Set(d, static_cast<T>(+3.14159265358979323846264));
const V kPiOverTwo = Set(d, static_cast<T>(+1.57079632679489661923132169));
const V sign_x = And(SignBit(d), x);
const V abs_x = Xor(x, sign_x);
const auto mask = Lt(abs_x, kHalf);
const V yy =
IfThenElse(mask, Mul(abs_x, abs_x), NegMulAdd(abs_x, kHalf, kHalf));
const V y = IfThenElse(mask, abs_x, Sqrt(yy));
impl::AsinImpl<T> impl;
const V t = Mul(impl.AsinPoly(d, yy, y), Mul(y, yy));
const V t_plus_y = Add(t, y);
const V z =
IfThenElse(mask, Sub(kPiOverTwo, Add(Xor(y, sign_x), Xor(t, sign_x))),
Add(t_plus_y, t_plus_y));
return IfThenElse(Or(mask, Ge(x, kZero)), z, Sub(kPi, z));
}
template <class D, class V>
inline __attribute__((always_inline)) V Acosh(const D d, V x) {
using T = TFromD<D>;
const V kLarge = Set(d, static_cast<T>(268435456.0));
const V kLog2 = Set(d, static_cast<T>(0.693147180559945286227));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kTwo = Set(d, static_cast<T>(+2.0));
const auto is_x_large = Gt(x, kLarge);
const auto is_x_gt_2 = Gt(x, kTwo);
const V x_minus_1 = Sub(x, kOne);
const V y0 = MulSub(kTwo, x, Div(kOne, Add(Sqrt(MulSub(x, x, kOne)), x)));
const V y1 =
Add(Sqrt(MulAdd(x_minus_1, kTwo, Mul(x_minus_1, x_minus_1))), x_minus_1);
const V y2 =
IfThenElse(is_x_gt_2, IfThenElse(is_x_large, x, y0), Add(y1, kOne));
const V z = impl::Log<D, V, false>(d, y2);
const auto is_pole = Eq(y2, kOne);
const auto divisor = Sub(IfThenZeroElse(is_pole, y2), kOne);
return Add(IfThenElse(is_x_gt_2, z,
IfThenElse(is_pole, y1, Div(Mul(z, y1), divisor))),
IfThenElseZero(is_x_large, kLog2));
}
template <class D, class V>
inline __attribute__((always_inline)) V Asin(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kTwo = Set(d, static_cast<T>(+2.0));
const V kPiOverTwo = Set(d, static_cast<T>(+1.57079632679489661923132169));
const V sign_x = And(SignBit(d), x);
const V abs_x = Xor(x, sign_x);
const auto mask = Lt(abs_x, kHalf);
const V yy =
IfThenElse(mask, Mul(abs_x, abs_x), NegMulAdd(abs_x, kHalf, kHalf));
const V y = IfThenElse(mask, abs_x, Sqrt(yy));
impl::AsinImpl<T> impl;
const V z0 = MulAdd(impl.AsinPoly(d, yy, y), Mul(yy, y), y);
const V z1 = NegMulAdd(z0, kTwo, kPiOverTwo);
return Or(IfThenElse(mask, z0, z1), sign_x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Asinh(const D d, V x) {
using T = TFromD<D>;
const V kSmall = Set(d, static_cast<T>(1.0 / 268435456.0));
const V kLarge = Set(d, static_cast<T>(268435456.0));
const V kLog2 = Set(d, static_cast<T>(0.693147180559945286227));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kTwo = Set(d, static_cast<T>(+2.0));
const V sign_x = And(SignBit(d), x);
const V abs_x = Xor(x, sign_x);
const auto is_x_large = Gt(abs_x, kLarge);
const auto is_x_lt_2 = Lt(abs_x, kTwo);
const V x2 = Mul(x, x);
const V sqrt_x2_plus_1 = Sqrt(Add(x2, kOne));
const V y0 = MulAdd(abs_x, kTwo, Div(kOne, Add(sqrt_x2_plus_1, abs_x)));
const V y1 = Add(Div(x2, Add(sqrt_x2_plus_1, kOne)), abs_x);
const V y2 =
IfThenElse(is_x_lt_2, Add(y1, kOne), IfThenElse(is_x_large, abs_x, y0));
const V z = impl::Log<D, V, false>(d, y2);
const auto is_pole = Eq(y2, kOne);
const auto divisor = Sub(IfThenZeroElse(is_pole, y2), kOne);
const auto large = IfThenElse(is_pole, y1, Div(Mul(z, y1), divisor));
const V y = IfThenElse(Lt(abs_x, kSmall), x, large);
return Or(Add(IfThenElse(is_x_lt_2, y, z), IfThenElseZero(is_x_large, kLog2)),
sign_x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Atan(const D d, V x) {
using T = TFromD<D>;
const V kOne = Set(d, static_cast<T>(+1.0));
const V kPiOverTwo = Set(d, static_cast<T>(+1.57079632679489661923132169));
const V sign = And(SignBit(d), x);
const V abs_x = Xor(x, sign);
const auto mask = Gt(abs_x, kOne);
impl::AtanImpl<T> impl;
const auto divisor = IfThenElse(mask, abs_x, kOne);
const V y = impl.AtanPoly(d, IfThenElse(mask, Div(kOne, divisor), abs_x));
return Or(IfThenElse(mask, Sub(kPiOverTwo, y), y), sign);
}
template <class D, class V>
inline __attribute__((always_inline)) V Atanh(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kOne = Set(d, static_cast<T>(+1.0));
const V sign = And(SignBit(d), x);
const V abs_x = Xor(x, sign);
return Mul(Log1p(d, Div(Add(abs_x, abs_x), Sub(kOne, abs_x))),
Xor(kHalf, sign));
}
template <class D, class V>
inline __attribute__((always_inline)) V Cos(const D d, V x) {
using T = TFromD<D>;
impl::CosSinImpl<T> impl;
const V kOneOverPi = Set(d, static_cast<T>(0.31830988618379067153));
const Rebind<int32_t, D> di32;
using VI32 = decltype(Zero(di32));
const VI32 kOne = Set(di32, 1);
const V y = Abs(x);
const VI32 q = Add(ShiftLeft<1>(impl.ToInt32(d, Mul(y, kOneOverPi))), kOne);
return impl.Poly(
d, Xor(impl.CosReduce(d, y, q), impl.CosSignFromQuadrant(d, q)));
}
template <class D, class V>
inline __attribute__((always_inline)) V Exp(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kLowerBound =
Set(d, static_cast<T>((sizeof(T) == 4 ? -104.0 : -1000.0)));
const V kNegZero = Set(d, static_cast<T>(-0.0));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kOneOverLog2 = Set(d, static_cast<T>(+1.442695040888963407359924681));
impl::ExpImpl<T> impl;
const auto q =
impl.ToInt32(d, MulAdd(x, kOneOverLog2, Or(kHalf, And(x, kNegZero))));
const V y = impl.LoadExpShortRange(
d, Add(impl.ExpPoly(d, impl.ExpReduce(d, x, q)), kOne), q);
return IfThenElseZero(Ge(x, kLowerBound), y);
}
template <class D, class V>
inline __attribute__((always_inline)) V Expm1(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kLowerBound =
Set(d, static_cast<T>((sizeof(T) == 4 ? -104.0 : -1000.0)));
const V kLn2Over2 = Set(d, static_cast<T>(+0.346573590279972654708616));
const V kNegOne = Set(d, static_cast<T>(-1.0));
const V kNegZero = Set(d, static_cast<T>(-0.0));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kOneOverLog2 = Set(d, static_cast<T>(+1.442695040888963407359924681));
impl::ExpImpl<T> impl;
const auto q =
impl.ToInt32(d, MulAdd(x, kOneOverLog2, Or(kHalf, And(x, kNegZero))));
const V y = impl.ExpPoly(d, impl.ExpReduce(d, x, q));
const V z = IfThenElse(Lt(Abs(x), kLn2Over2), y,
Sub(impl.LoadExpShortRange(d, Add(y, kOne), q), kOne));
return IfThenElse(Lt(x, kLowerBound), kNegOne, z);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log(const D d, V x) {
return impl::Log<D, V, true>(d, x);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log10(const D d, V x) {
using T = TFromD<D>;
return Mul(Log(d, x), Set(d, static_cast<T>(0.4342944819032518276511)));
}
template <class D, class V>
inline __attribute__((always_inline)) V Log1p(const D d, V x) {
using T = TFromD<D>;
const V kOne = Set(d, static_cast<T>(+1.0));
const V y = Add(x, kOne);
const auto is_pole = Eq(y, kOne);
const auto divisor = Sub(IfThenZeroElse(is_pole, y), kOne);
const auto non_pole =
Mul(impl::Log<D, V, false>(d, y), Div(x, divisor));
return IfThenElse(is_pole, x, non_pole);
}
template <class D, class V>
inline __attribute__((always_inline)) V Log2(const D d, V x) {
using T = TFromD<D>;
return Mul(Log(d, x), Set(d, static_cast<T>(1.44269504088896340735992)));
}
template <class D, class V>
inline __attribute__((always_inline)) V Sin(const D d, V x) {
using T = TFromD<D>;
impl::CosSinImpl<T> impl;
const V kOneOverPi = Set(d, static_cast<T>(0.31830988618379067153));
const V kHalf = Set(d, static_cast<T>(0.5));
const Rebind<int32_t, D> di32;
using VI32 = decltype(Zero(di32));
const V abs_x = Abs(x);
const V sign_x = Xor(abs_x, x);
const VI32 q = impl.ToInt32(d, MulAdd(abs_x, kOneOverPi, kHalf));
return impl.Poly(d, Xor(impl.SinReduce(d, abs_x, q),
Xor(impl.SinSignFromQuadrant(d, q), sign_x)));
}
template <class D, class V>
inline __attribute__((always_inline)) V Sinh(const D d, V x) {
using T = TFromD<D>;
const V kHalf = Set(d, static_cast<T>(+0.5));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kTwo = Set(d, static_cast<T>(+2.0));
const V sign = And(SignBit(d), x);
const V abs_x = Xor(x, sign);
const V y = Expm1(d, abs_x);
const V z = Mul(Div(Add(y, kTwo), Add(y, kOne)), Mul(y, kHalf));
return Xor(z, sign);
}
template <class D, class V>
inline __attribute__((always_inline)) V Tanh(const D d, V x) {
using T = TFromD<D>;
const V kLimit = Set(d, static_cast<T>(18.714973875));
const V kOne = Set(d, static_cast<T>(+1.0));
const V kTwo = Set(d, static_cast<T>(+2.0));
const V sign = And(SignBit(d), x);
const V abs_x = Xor(x, sign);
const V y = Expm1(d, Mul(abs_x, kTwo));
const V z = IfThenElse(Gt(abs_x, kLimit), kOne, Div(y, Add(y, kTwo)));
return Xor(z, sign);
}
}
}
static_assert(true, "For requiring trailing semicolon");
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_EMU128 {
template <class D, class V = VFromD<D>>
static inline __attribute__((always_inline)) __attribute__((flatten)) __attribute__((unused)) void Print(const D d, const char* caption, V v, size_t lane_u = 0,
size_t max_lanes = 7) {
const size_t N = Lanes(d);
using T = TFromD<D>;
auto lanes = AllocateAligned<T>(N);
Store(v, d, lanes.get());
const auto info = hwy::detail::MakeTypeInfo<T>();
hwy::detail::PrintArray(info, caption, lanes.get(), N, lane_u, max_lanes);
}
}
}
static_assert(true, "For requiring trailing semicolon");
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_EMU128 {
template <class D, typename T = TFromD<D>>
inline __attribute__((always_inline)) void AssertVecEqual(D d, const T* expected, Vec<D> actual,
const char* filename, const int line) {
const size_t N = Lanes(d);
auto actual_lanes = AllocateAligned<T>(N);
Store(actual, d, actual_lanes.get());
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName((((((1LL << 61) | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0)))) & -((((1LL << 61) | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0))))));
hwy::detail::AssertArrayEqual(info, expected, actual_lanes.get(), N,
target_name, filename, line);
}
template <class D, typename T = TFromD<D>>
inline __attribute__((always_inline)) void AssertVecEqual(D d, Vec<D> expected, Vec<D> actual,
const char* filename, int line) {
const size_t N = Lanes(d);
auto expected_lanes = AllocateAligned<T>(N);
auto actual_lanes = AllocateAligned<T>(N);
Store(expected, d, expected_lanes.get());
Store(actual, d, actual_lanes.get());
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName((((((1LL << 61) | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0)))) & -((((1LL << 61) | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0))))));
hwy::detail::AssertArrayEqual(info, expected_lanes.get(), actual_lanes.get(),
N, target_name, filename, line);
}
template <class D>
__attribute__((noinline)) void AssertMaskEqual(D d, VecArg<Mask<D>> a, VecArg<Mask<D>> b,
const char* filename, int line) {
const Vec<D> va = VecFromMask(d, a);
const Vec<D> vb = VecFromMask(d, b);
AssertVecEqual(d, va, vb, filename, line);
const char* target_name = hwy::TargetName((((((1LL << 61) | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0)))) & -((((1LL << 61) | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0)) & ~((0) | ((0 | ((1LL << 9) | (1LL << 8) | (1LL << 7) | (1LL << 6)) | 0 | 0 | 0 | 0 | 0 | 0))))));
AssertEqual(CountTrue(d, a), CountTrue(d, b), target_name, filename, line);
AssertEqual(AllTrue(d, a), AllTrue(d, b), target_name, filename, line);
AssertEqual(AllFalse(d, a), AllFalse(d, b), target_name, filename, line);
const size_t N = Lanes(d);
const Repartition<uint8_t, D> d8;
const size_t N8 = Lanes(d8);
auto bits_a = AllocateAligned<uint8_t>(((size_t{8}) > (N8) ? (size_t{8}) : (N8)));
auto bits_b = AllocateAligned<uint8_t>(size_t{((8) > (N8) ? (8) : (N8))});
memset(bits_a.get(), 0, N8);
memset(bits_b.get(), 0, N8);
const size_t num_bytes_a = StoreMaskBits(d, a, bits_a.get());
const size_t num_bytes_b = StoreMaskBits(d, b, bits_b.get());
AssertEqual(num_bytes_a, num_bytes_b, target_name, filename, line);
size_t i = 0;
for (; i < N / 8; ++i) {
if (bits_a[i] != bits_b[i]) {
fprintf(
stderr
, "Mismatch in byte %d: %d != %d\n", static_cast<int>(i),
bits_a[i], bits_b[i]);
Print(d8, "expect", Load(d8, bits_a.get()), 0, N8);
Print(d8, "actual", Load(d8, bits_b.get()), 0, N8);
hwy::Abort(filename, line, "Masks not equal");
}
}
const size_t remainder = N % 8;
if (remainder != 0) {
const int mask = (1 << remainder) - 1;
const int valid_a = bits_a[i] & mask;
const int valid_b = bits_b[i] & mask;
if (valid_a != valid_b) {
fprintf(
stderr
, "Mismatch in last byte %d: %d != %d\n",
static_cast<int>(i), valid_a, valid_b);
Print(d8, "expect", Load(d8, bits_a.get()), 0, N8);
Print(d8, "actual", Load(d8, bits_b.get()), 0, N8);
hwy::Abort(filename, line, "Masks not equal");
}
}
}
template <class D>
inline __attribute__((always_inline)) Mask<D> MaskTrue(const D d) {
return FirstN(d, Lanes(d));
}
template <class D>
inline __attribute__((always_inline)) Mask<D> MaskFalse(const D d) {
const RebindToSigned<D> di;
const Vec<decltype(di)> zero = Zero(di);
return RebindMask(d, Lt(zero, zero));
}
namespace detail {
template <typename T, size_t kMul, size_t kMinArg, class Test, int kPow2 = 0>
struct ForeachCappedR {
static void Do(size_t min_lanes, size_t max_lanes) {
const CappedTag<T, kMul * kMinArg, kPow2> d;
const size_t lanes = Lanes(d);
if (lanes < min_lanes) return;
if (lanes <= max_lanes) {
Test()(T(), d);
}
ForeachCappedR<T, kMul / 2, kMinArg, Test, kPow2>::Do(min_lanes, max_lanes);
}
};
template <typename T, size_t kMinArg, class Test, int kPow2>
struct ForeachCappedR<T, 0, kMinArg, Test, kPow2> {
static void Do(size_t, size_t) {}
};
}
template <class Test>
class ForMaxPow2 {
mutable bool called_ = false;
public:
~ForMaxPow2() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 291, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
detail::ForeachCappedR<T, (16 / sizeof(T)), 1, Test>::Do(
1, Lanes(ScalableTag<T>()));
}
};
template <class Test, int kPow2 = 1>
class ForExtendableVectors {
mutable bool called_ = false;
public:
~ForExtendableVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 324, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMaxCapped = (16 / sizeof(T));
const size_t max_lanes = Lanes(ScalableTag<T>()) >> kPow2;
(void)kMaxCapped;
(void)max_lanes;
constexpr size_t kMul = kMaxCapped >> kPow2;
constexpr size_t kMinArg = size_t{1} << kPow2;
detail::ForeachCappedR<T, kMul, kMinArg, Test, -kPow2>::Do(1, max_lanes);
}
};
template <class Test, int kPow2 = 1>
class ForShrinkableVectors {
mutable bool called_ = false;
public:
~ForShrinkableVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 358, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMinLanes = size_t{1} << kPow2;
constexpr size_t kMaxCapped = (16 / sizeof(T));
constexpr size_t max_lanes = kMaxCapped;
(void)kMinLanes;
(void)max_lanes;
(void)max_lanes;
detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kMinLanes, Test>::Do(
kMinLanes, max_lanes);
}
};
template <size_t kMinBits, class Test>
class ForGEVectors {
mutable bool called_ = false;
public:
~ForGEVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 394, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMaxCapped = (16 / sizeof(T));
constexpr size_t kMinLanes = kMinBits / 8 / sizeof(T);
constexpr size_t max_lanes = kMaxCapped;
(void)max_lanes;
detail::ForeachCappedR<T, (16 / sizeof(T)) / kMinLanes, kMinLanes, Test>::Do(
kMinLanes, max_lanes);
}
};
template <class Test>
using ForGE128Vectors = ForGEVectors<128, Test>;
template <class Test, int kPow2 = 1>
class ForPromoteVectors {
mutable bool called_ = false;
public:
~ForPromoteVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 436, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kFactor = size_t{1} << kPow2;
static_assert(kFactor >= 2 && kFactor * sizeof(T) <= sizeof(uint64_t), "");
constexpr size_t kMaxCapped = (16 / sizeof(T));
const size_t max_lanes = Lanes(ScalableTag<T>()) >> kPow2;
(void)kMaxCapped;
(void)max_lanes;
using DLargestFrom = CappedTag<T, (kMaxCapped >> kPow2) * kFactor, -kPow2>;
static_assert(DLargestFrom::kPrivateLanes <= (kMaxCapped >> kPow2),
"HWY_MAX_LANES_D(DLargestFrom) must be less than or equal to "
"(kMaxCapped >> kPow2)");
detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kFactor, Test, -kPow2>::Do(
1, max_lanes);
}
};
template <class Test, int kPow2 = 1>
class ForDemoteVectors {
mutable bool called_ = false;
public:
~ForDemoteVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 475, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMinLanes = size_t{1} << kPow2;
constexpr size_t kMaxCapped = (16 / sizeof(T));
constexpr size_t max_lanes = kMaxCapped;
(void)kMinLanes;
(void)max_lanes;
(void)max_lanes;
detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kMinLanes, Test>::Do(
kMinLanes, max_lanes);
}
};
template <class Test, int kPow2 = 1>
class ForHalfVectors {
mutable bool called_ = false;
public:
~ForHalfVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 512, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T ) const {
called_ = true;
constexpr size_t kMinLanes = size_t{1} << kPow2;
constexpr size_t kMaxCapped = (16 / sizeof(T));
detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kMinLanes, Test>::Do(
kMinLanes, kMaxCapped);
}
};
template <class Test>
class ForPartialVectors {
mutable bool called_ = false;
public:
~ForPartialVectors() {
if (!called_) {
::hwy::Abort("/home/mathieu/Perso/highway/hwy/tests/test_util-inl.h", 545, "Test is incorrect, ensure operator() is called");
}
}
template <typename T>
void operator()(T t) const {
called_ = true;
ForExtendableVectors<Test, 0>()(t);
}
};
template <class Test>
using ForPartialFixedOrFullScalableVectors = ForPartialVectors<Test>;
template <class Func>
void ForSignedTypes(const Func& func) {
func(int8_t());
func(int16_t());
func(int32_t());
func(int64_t());
}
template <class Func>
void ForUnsignedTypes(const Func& func) {
func(uint8_t());
func(uint16_t());
func(uint32_t());
func(uint64_t());
}
template <class Func>
void ForIntegerTypes(const Func& func) {
ForSignedTypes(func);
ForUnsignedTypes(func);
}
template <class Func>
void ForFloatTypes(const Func& func) {
func(float());
func(double());
}
template <class Func>
void ForAllTypes(const Func& func) {
ForIntegerTypes(func);
ForFloatTypes(func);
}
template <class Func>
void ForUI8(const Func& func) {
func(uint8_t());
func(int8_t());
}
template <class Func>
void ForUI16(const Func& func) {
func(uint16_t());
func(int16_t());
}
template <class Func>
void ForUIF16(const Func& func) {
ForUI16(func);
func(float16_t());
}
template <class Func>
void ForUI32(const Func& func) {
func(uint32_t());
func(int32_t());
}
template <class Func>
void ForUIF32(const Func& func) {
ForUI32(func);
func(float());
}
template <class Func>
void ForUI64(const Func& func) {
func(uint64_t());
func(int64_t());
}
template <class Func>
void ForUIF64(const Func& func) {
ForUI64(func);
func(double());
}
template <class Func>
void ForUI3264(const Func& func) {
ForUI32(func);
ForUI64(func);
}
template <class Func>
void ForUIF3264(const Func& func) {
ForUIF32(func);
ForUIF64(func);
}
template <class Func>
void ForU163264(const Func& func) {
func(uint16_t());
func(uint32_t());
func(uint64_t());
}
template <class Func>
void ForUI163264(const Func& func) {
ForUI16(func);
ForUI3264(func);
}
template <class Func>
void ForUIF163264(const Func& func) {
ForUIF16(func);
ForUIF3264(func);
}
constexpr size_t AdjustedReps(size_t max_reps) {
return ((max_reps) > (2) ? (max_reps) : (2));
}
constexpr size_t AdjustedLog2Reps(size_t max_pow2) {
return max_pow2;
}
}
}
static_assert(true, "For requiring trailing semicolon");
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
namespace N_EMU128 {
template <class Out, class In>
inline Out BitCast(const In& in) {
static_assert(sizeof(Out) == sizeof(In), "");
Out out;
CopyBytes<sizeof(out)>(&in, &out);
return out;
}
template <class T, class D>
__attribute__((noinline)) void TestMath(const char* name, T (*fx1)(T),
Vec<D> (*fxN)(D, VecArg<Vec<D>>), D d, T min, T max,
uint64_t max_error_ulp) {
using UintT = MakeUnsigned<T>;
const UintT min_bits = BitCast<UintT>(min);
const UintT max_bits = BitCast<UintT>(max);
int range_count = 1;
UintT ranges[2][2] = {{min_bits, max_bits}, {0, 0}};
if ((min < 0.0) && (max > 0.0)) {
ranges[0][0] = BitCast<UintT>(static_cast<T>(+0.0));
ranges[0][1] = max_bits;
ranges[1][0] = BitCast<UintT>(static_cast<T>(-0.0));
ranges[1][1] = min_bits;
range_count = 2;
}
uint64_t max_ulp = 0;
constexpr UintT kSamplesPerRange = static_cast<UintT>(AdjustedReps(4000));
for (int range_index = 0; range_index < range_count; ++range_index) {
const UintT start = ranges[range_index][0];
const UintT stop = ranges[range_index][1];
const UintT step = ((1) > (((stop - start) / kSamplesPerRange)) ? (1) : (((stop - start) / kSamplesPerRange)));
for (UintT value_bits = start; value_bits <= stop; value_bits += step) {
const T value = BitCast<T>(((((start) > (value_bits) ? (start) : (value_bits))) < (stop) ? (((start) > (value_bits) ? (start) : (value_bits))) : (stop)));
const T actual = GetLane(fxN(d, Set(d, value)));
const T expected = fx1(value);
const auto ulp = hwy::detail::ComputeUlpDelta(actual, expected);
max_ulp = ((max_ulp) > (ulp) ? (max_ulp) : (ulp));
if (ulp > max_error_ulp) {
fprintf(
stderr
, "KO %s: %s(%.17g) expected %.17g actual %.17g ulp %g max ulp %u\n",
hwy::TypeName(T(), Lanes(d)).c_str(), name, value, expected,
actual, static_cast<double>(ulp),
static_cast<uint32_t>(max_error_ulp));
if( value == 5.526720574244119e-20 && expected == 5.526720574244119e-20 && actual == 0) exit(42);
}
else {
fprintf(
stderr
, "OK %s: %s(%.17g) expected %.17g actual %.17g ulp %g max ulp %u\n",
hwy::TypeName(T(), Lanes(d)).c_str(), name, value, expected,
actual, static_cast<double>(ulp),
static_cast<uint32_t>(max_error_ulp));
}
}
}
fprintf(
stderr
, "%s: %s max_ulp %g\n", hwy::TypeName(T(), Lanes(d)).c_str(),
name, static_cast<double>(max_ulp));
}
const float kNearOneF = BitCast<float>(0x3F7FFFFF);
const double kNearOneD = BitCast<double>(0x3FEFFFFFFFFFFFFFULL);
struct TestLog1p { template <class T, class D> __attribute__((noinline)) void operator()(T, D d) { if (sizeof(T) == 4) { TestMath<T, D>("Log1p", std::log1p, CallLog1p, d, +0.0f, +1e37f, 3); } else { TestMath<T, D>("Log1p", std::log1p, CallLog1p, d, static_cast<T>(+0.0), static_cast<T>(+double(1.79769313486231570814527423731704357e+308L)
), 2); } } }; __attribute__((noinline)) void TestAllLog1p() { ForFloatTypes(ForPartialVectors<TestLog1p>()); }
}
}
static_assert(true, "For requiring trailing semicolon");
namespace hwy {
static decltype(&N_EMU128::TestAllLog1p) const TestAllLog1pHighwayDispatchTable[15 + 2] = { &decltype(hwy::DeduceFunctionCache(&N_EMU128::TestAllLog1p))::ChooseAndCall<TestAllLog1pHighwayDispatchTable>, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, &N_SSE4::TestAllLog1p, &N_SSSE3::TestAllLog1p, nullptr, &N_SSE2::TestAllLog1p, &N_EMU128::TestAllLog1p, };
void Run() {
for (int64_t target : SupportedAndGeneratedTargets()) {
SetSupportedTargetsForTest(target);
(*(TestAllLog1pHighwayDispatchTable[hwy::GetChosenTarget().GetIndex()]))();
}
SetSupportedTargetsForTest(0);
}
}
int main(int , char** ) {
hwy::Run();
return 0;
}